修复 AsyncSocket 的 bug

This commit is contained in:
tursom 2019-11-03 23:22:39 +08:00
parent 5af7493ff5
commit 9765c5ead5
21 changed files with 1443 additions and 1132 deletions

View File

@ -1,8 +1,10 @@
package cn.tursom.socket
import cn.tursom.socket.niothread.INioThread
import cn.tursom.core.log
import cn.tursom.core.logE
import cn.tursom.core.timer.TimerTask
import cn.tursom.core.timer.WheelTimer
import cn.tursom.socket.niothread.INioThread
import java.nio.ByteBuffer
import java.nio.channels.SelectionKey
import java.nio.channels.SocketChannel
@ -23,7 +25,9 @@ class AsyncNioSocket(override val key: SelectionKey, override val nioThread: INi
override suspend fun read(buffer: ByteBuffer): Int {
if (buffer.remaining() == 0) return emptyBufferCode
return operate {
//logE("read(buffer: ByteBuffer) wait read")
waitRead()
//logE("read(buffer: ByteBuffer) wait read complete")
channel.read(buffer)
}
}
@ -53,10 +57,13 @@ class AsyncNioSocket(override val key: SelectionKey, override val nioThread: INi
}
override suspend fun read(buffer: ByteBuffer, timeout: Long): Int {
//logE("AsyncNioSocket.read(buffer: ByteBuffer, timeout: Long): $buffer, $timeout")
if (timeout <= 0) return read(buffer)
if (buffer.remaining() == 0) return emptyBufferCode
return operate {
//logE("wait read")
waitRead(timeout)
//logE("wait read complete")
channel.read(buffer)
}
}
@ -109,10 +116,11 @@ class AsyncNioSocket(override val key: SelectionKey, override val nioThread: INi
suspendCoroutine<Int> {
key.attach(Context(it, timer.exec(timeout) {
key.attach(null)
waitMode()
it.resumeWithException(TimeoutException())
}))
readMode()
nioThread.wakeup()
}))
}
}
@ -120,17 +128,21 @@ class AsyncNioSocket(override val key: SelectionKey, override val nioThread: INi
suspendCoroutine<Int> {
key.attach(Context(it, timer.exec(timeout) {
key.attach(null)
waitMode()
it.resumeWithException(TimeoutException())
}))
writeMode()
nioThread.wakeup()
}))
}
}
private suspend inline fun waitRead() {
suspendCoroutine<Int> {
//logE("waitRead() attach")
key.attach(Context(it))
//logE("waitRead() readMode()")
readMode()
//logE("waitRead() wakeup()")
nioThread.wakeup()
}
}
@ -151,6 +163,7 @@ class AsyncNioSocket(override val key: SelectionKey, override val nioThread: INi
override fun handleRead(key: SelectionKey, nioThread: INioThread) {
key.interestOps(0)
//logE("read ready")
val context = key.attachment() as Context? ?: return
context.timeoutTask?.cancel()
context.cont.resume(0)
@ -159,6 +172,7 @@ class AsyncNioSocket(override val key: SelectionKey, override val nioThread: INi
override fun handleWrite(key: SelectionKey, nioThread: INioThread) {
key.interestOps(0)
val context = key.attachment() as Context? ?: return
context.timeoutTask?.cancel()
context.cont.resume(0)
}

View File

@ -3,19 +3,21 @@ package cn.tursom.socket
import cn.tursom.core.bytebuffer.AdvanceByteBuffer
import cn.tursom.core.bytebuffer.readNioBuffer
import cn.tursom.core.bytebuffer.writeNioBuffer
import cn.tursom.core.logE
import java.io.Closeable
import java.nio.ByteBuffer
interface AsyncSocket : Closeable {
suspend fun write(buffer: Array<out ByteBuffer>, timeout: Long = 0L): Long
suspend fun read(buffer: Array<out ByteBuffer>, timeout: Long = 0L): Long
suspend fun write(buffer: ByteBuffer, timeout: Long = 0L): Int = write(arrayOf(buffer)).toInt()
suspend fun read(buffer: ByteBuffer, timeout: Long = 0L): Int = read(arrayOf(buffer)).toInt()
suspend fun write(buffer: ByteBuffer, timeout: Long = 0L): Int = write(arrayOf(buffer), timeout).toInt()
suspend fun read(buffer: ByteBuffer, timeout: Long = 0L): Int = read(arrayOf(buffer), timeout).toInt()
override fun close()
suspend fun write(buffer: AdvanceByteBuffer, timeout: Long = 0): Int {
return if (buffer.bufferCount == 1) {
buffer.readNioBuffer {
//logE(it.toString())
write(it, timeout)
}
} else {
@ -28,6 +30,8 @@ interface AsyncSocket : Closeable {
}
suspend fun read(buffer: AdvanceByteBuffer, timeout: Long = 0): Int {
//logE("buffer.bufferCount: ${buffer.bufferCount}")
//logE("AsyncSocket.read(buffer: AdvanceByteBuffer, timeout: Long = 0): buffer: $buffer")
return if (buffer.bufferCount == 1) {
buffer.writeNioBuffer {
read(it, timeout)

View File

@ -3,6 +3,7 @@ package cn.tursom.socket
import cn.tursom.socket.niothread.INioThread
import cn.tursom.core.bytebuffer.AdvanceByteBuffer
import cn.tursom.core.bytebuffer.writeNioBuffer
import cn.tursom.core.logE
import java.net.SocketException
import java.nio.ByteBuffer
import java.nio.channels.SelectionKey
@ -23,10 +24,15 @@ interface IAsyncNioSocket : AsyncSocket {
}
fun readMode() {
//logE("readMode()")
if (Thread.currentThread() == nioThread.thread) {
if (key.isValid) key.interestOps(SelectionKey.OP_WRITE)
} else {
nioThread.execute { if (key.isValid) key.interestOps(SelectionKey.OP_READ) }
nioThread.execute {
//logE("readMode() interest")
if (key.isValid) key.interestOps(SelectionKey.OP_READ)
//logE("readMode interestOps ${key.isValid} ${key.interestOps()}")
}
nioThread.wakeup()
}
}

View File

@ -23,7 +23,7 @@ class AsyncNioServer(
try {
socket.handler()
} catch (e: Exception) {
e.printStackTrace()
Exception(e).printStackTrace()
} finally {
try {
socket.close()

View File

@ -0,0 +1,93 @@
import cn.tursom.core.bytebuffer.ByteArrayAdvanceByteBuffer
import cn.tursom.core.log
import cn.tursom.core.pool.DirectMemoryPool
import cn.tursom.core.pool.usingAdvanceByteBuffer
import cn.tursom.socket.AsyncNioClient
import cn.tursom.socket.server.AsyncNioServer
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
import java.lang.Thread.sleep
import java.net.SocketException
import java.util.concurrent.TimeoutException
import java.util.concurrent.atomic.AtomicInteger
fun main() {
// 服务器端口,可任意指定
val port = 12345
// 创建一个直接内存池每个块是1024字节共有256个快
val memoryPool = DirectMemoryPool(1024, 256)
// 创建服务器对象
val server = AsyncNioServer(port) {
// 这里处理业务逻辑,套接字对象被以 this 的方式传进来
// 从内存池中获取一个内存块
memoryPool.usingAdvanceByteBuffer {
// 检查是否获取成功,不成功就创建一个堆缓冲
val buffer = it ?: ByteArrayAdvanceByteBuffer(1024)
try {
while (true) {
buffer.clear()
// 从套接字中读数据,五秒之内没有数据就抛出异常
if (read(buffer, 10_000) < 0) {
return@AsyncNioServer
}
// 输出读取到的数据
//log("server recv from ${channel.remoteAddress}: [${buffer.readableSize}] ${buffer.toString(buffer.readableSize)}")
// 原封不动的返回数据
val writeSize = write(buffer)
//log("server send [$writeSize] bytes")
}
} catch (e: TimeoutException) {
}
// 代码块结束后,框架会自动释放连接
}
}
server.run()
val connectionCount = 300
val dataPerConn = 10
val testData = "testData".toByteArray()
val remain = AtomicInteger(connectionCount)
val clientMemoryPool = DirectMemoryPool(1024, connectionCount)
val start = System.currentTimeMillis()
repeat(connectionCount) {
GlobalScope.launch {
val socket = AsyncNioClient.connect("127.0.0.1", port)
clientMemoryPool.usingAdvanceByteBuffer {
// 检查是否获取成功,不成功就创建一个堆缓冲
val buffer = it ?: ByteArrayAdvanceByteBuffer(1024)
try {
repeat(dataPerConn) {
buffer.clear()
buffer.put(testData)
//log("client sending: [${buffer.readableSize}] ${buffer.toString(buffer.readableSize)}")
val writeSize = socket.write(buffer)
//log("client write [$writeSize] bytes")
//log(buffer.toString())
val readSize = socket.read(buffer)
//log(buffer.toString())
//log("client recv: [$readSize:${buffer.readableSize}] ${buffer.toString(buffer.readableSize)}")
}
} catch (e: Exception) {
Exception(e).printStackTrace()
} finally {
socket.close()
}
}
remain.decrementAndGet()
}
}
while (remain.get() != 0) {
println(remain.get())
sleep(500)
}
val end = System.currentTimeMillis()
println(end - start)
server.close()
}

View File

@ -0,0 +1,59 @@
package cn.tursom.socket.server
import cn.tursom.core.bytebuffer.ByteArrayAdvanceByteBuffer
import cn.tursom.core.log
import cn.tursom.core.logE
import cn.tursom.socket.AsyncAioClient
import cn.tursom.socket.SocketClient
import kotlinx.coroutines.runBlocking
import org.junit.Test
class AsyncNioServerTest {
private val testMsg = "hello"
private val port = 12345
private val server = AsyncNioServer(port) {
log("new connection")
val buffer = ByteArrayAdvanceByteBuffer(1024)
while (true) {
buffer.clear()
read(buffer, 5000)
logE("server recv: ${buffer.toString(buffer.readableSize)}")
write(buffer, 5000)
}
}
init {
server.run()
}
//@Test
fun testAsyncNioServer() {
runBlocking {
val client = AsyncAioClient.connect("127.0.0.1", port)
log("connect to server")
val buffer = ByteArrayAdvanceByteBuffer(1024)
repeat(10) {
buffer.clear()
buffer.put(testMsg)
client.write(buffer, 5000)
buffer.clear()
client.read(buffer, 5000)
log("server recv: ${buffer.getString()}")
}
}
}
//@Test
fun testAsyncNioServerSocket() {
SocketClient("localhost", port) {
val buffer = ByteArray(1024)
repeat(10) {
send(testMsg)
val readSize = inputStream.read(buffer)
val recv = String(buffer, 0, readSize)
log(recv)
}
}
}
}

View File

@ -16,8 +16,8 @@ open class BaseSocket(
val address = socket.inetAddress?.toString()?.drop(1) ?: "0.0.0.0"
val port = socket.port
val localPort = socket.localPort
val inputStream = socket.getInputStream()!!
val outputStream = socket.getOutputStream()!!
val inputStream by lazy { socket.getInputStream()!! }
val outputStream by lazy { socket.getOutputStream()!! }
fun send(message: String?) {
send((message ?: return).toByteArray())

View File

@ -14,7 +14,7 @@ class SocketClient(
init {
func?.let {
use(it)
invoke(it)
}
}
@ -43,7 +43,7 @@ class SocketClient(
}
}
fun use(func: SocketClient.() -> Unit) {
operator fun invoke(func: SocketClient.() -> Unit) {
val ret = execute(func)
closeSocket()
return ret

View File

@ -1,5 +1,7 @@
package cn.tursom.socket.server
import cn.tursom.core.log
import cn.tursom.core.logE
import cn.tursom.socket.INioProtocol
import cn.tursom.socket.niothread.INioThread
import cn.tursom.socket.niothread.WorkerLoopNioThread
@ -48,6 +50,7 @@ class NioServer(
class LoopHandler(val protocol: INioProtocol) {
fun handle(nioThread: INioThread) {
//logE("wake up")
val selector = nioThread.selector
if (selector.isOpen) {
if (selector.select(TIMEOUT) != 0) {
@ -55,6 +58,7 @@ class NioServer(
while (keyIter.hasNext()) run whileBlock@{
val key = keyIter.next()
keyIter.remove()
//logE("selected key: $key: ${key.attachment()}")
try {
when {
key.isAcceptable -> {

View File

@ -0,0 +1,66 @@
package cn.tursom.socket.server
import cn.tursom.core.bytebuffer.AdvanceByteBuffer
import cn.tursom.core.bytebuffer.ByteArrayAdvanceByteBuffer
import cn.tursom.core.bytebuffer.readNioBuffer
import cn.tursom.core.bytebuffer.writeNioBuffer
import cn.tursom.core.pool.DirectMemoryPool
import cn.tursom.core.pool.MemoryPool
import cn.tursom.socket.INioProtocol
import cn.tursom.socket.SocketClient
import cn.tursom.socket.niothread.INioThread
import org.junit.Test
import java.nio.channels.SelectionKey
import java.nio.channels.SocketChannel
class NioServerTest {
private val port = 12345
@Test
fun testNioServer() {
val memoryPool: MemoryPool = DirectMemoryPool(1024, 256)
val server = NioServer(port, object : INioProtocol {
override fun handleConnect(key: SelectionKey, nioThread: INioThread) {
val memoryToken = memoryPool.allocate()
key.attach(memoryToken to (memoryPool.getAdvanceByteBuffer(memoryToken) ?: ByteArrayAdvanceByteBuffer(1024)))
key.interestOps(SelectionKey.OP_READ)
}
override fun handleRead(key: SelectionKey, nioThread: INioThread) {
val channel = key.channel() as SocketChannel
val buffer = (key.attachment() as Pair<Int, AdvanceByteBuffer>).second
buffer.writeNioBuffer {
channel.read(it)
}
println("record from client: ${buffer.toString(buffer.readableSize)}")
key.interestOps(SelectionKey.OP_WRITE)
}
override fun handleWrite(key: SelectionKey, nioThread: INioThread) {
val channel = key.channel() as SocketChannel
val buffer = (key.attachment() as Pair<Int, AdvanceByteBuffer>).second
println("send to client: ${buffer.toString(buffer.readableSize)}")
buffer.readNioBuffer {
channel.write(it)
}
buffer.reset()
key.interestOps(SelectionKey.OP_READ)
}
override fun exceptionCause(key: SelectionKey, nioThread: INioThread, e: Throwable) {
super.exceptionCause(key, nioThread, e)
val memoryToken = (key.attachment() as Pair<Int, AdvanceByteBuffer>).first
memoryPool.free(memoryToken)
key.channel().close()
key.cancel()
}
})
server.run()
val socket = SocketClient("127.0.0.1", port)
val buffer = ByteArray(1024)
socket.outputStream.write("Hello".toByteArray())
val readCount = socket.inputStream.read(buffer)
println(buffer.copyOfRange(0, readCount).toString(Charsets.UTF_8))
server.close()
}
}

View File

@ -1,6 +1,7 @@
package cn.tursom.core.bytebuffer
import cn.tursom.core.forEachIndex
import cn.tursom.core.logE
import sun.reflect.generics.reflectiveObjects.NotImplementedException
import java.io.OutputStream
import java.nio.ByteBuffer
@ -162,14 +163,14 @@ interface AdvanceByteBuffer {
}
fun getBytes(): ByteArray = if (readMode) {
val bytes = ByteArray(readableSize)
fun getBytes(size: Int = readableSize): ByteArray = if (readMode) {
val bytes = ByteArray(size)
nioBuffer.get(bytes)
readPosition = writePosition
bytes
} else {
readMode()
val bytes = ByteArray(readableSize)
val bytes = ByteArray(size)
nioBuffer.get(bytes)
readPosition = writePosition
resumeWriteMode()
@ -179,9 +180,11 @@ interface AdvanceByteBuffer {
fun getString(size: Int = readableSize): String = String(getBytes())
fun toString(size: Int): String {
val rp = readPosition
val bytes = getBytes()
readPosition = rp
//logE("AdvanceByteBuffer.toString(size: Int): $this")
//val rp = readPosition
val bytes = getBytes(size)
//readPosition = rp
//logE("AdvanceByteBuffer.toString(size: Int): $this")
return String(bytes)
}
@ -310,16 +313,10 @@ interface AdvanceByteBuffer {
}
fun put(str: String) {
if (readMode) {
resumeWriteMode()
nioBuffer.put(str.toByteArray())
readMode()
} else {
nioBuffer.put(str.toByteArray())
}
put(str.toByteArray())
}
fun put(byteArray: ByteArray, startIndex: Int = 0, endIndex: Int = byteArray.size) {
fun put(byteArray: ByteArray, startIndex: Int = 0, endIndex: Int = byteArray.size - startIndex) {
if (readMode) {
resumeWriteMode()
nioBuffer.put(byteArray, startIndex, endIndex - startIndex)
@ -409,27 +406,40 @@ interface AdvanceByteBuffer {
throw NotImplementedException()
}
}
override fun toString(): String
}
inline fun <T> AdvanceByteBuffer.readNioBuffer(action: (nioBuffer: ByteBuffer) -> T): T {
val readMode = this.readMode
readMode()
val buffer = nioBuffer
val position = nioBuffer.position()
val position = readPosition
val bufferPosition = nioBuffer.position()
return try {
//logE(buffer.toString())
action(buffer)
} finally {
if (!readMode) {
resumeWriteMode(buffer.position() - position)
readPosition = position + (buffer.position() - bufferPosition)
}
}
}
inline fun <T> AdvanceByteBuffer.writeNioBuffer(action: (nioBuffer: ByteBuffer) -> T): T {
val readMode = readMode
resumeWriteMode()
val buffer = nioBuffer
val position = writePosition
val bufferPosition = nioBuffer.position()
return try {
action(buffer)
} finally {
if (readMode) {
writePosition = position + (buffer.position() - bufferPosition)
readMode()
}
}
}

View File

@ -112,7 +112,14 @@ class ByteArrayAdvanceByteBuffer(
override fun getLong() = array.toLong(take(8))
override fun getFloat() = array.toFloat(take(4))
override fun getDouble() = array.toDouble(take(8))
override fun getBytes() = array.copyOfRange(readPosition, readAllSize())
override fun getBytes(size: Int): ByteArray {
val readMode = readMode
readMode()
val array = array.copyOfRange(readPosition, useReadSize(size))
if (!readMode) resumeWriteMode(size)
return array
}
override fun getString(size: Int) = String(array, readPosition, useReadSize(size))
override fun writeTo(buffer: ByteArray, bufferOffset: Int, size: Int): Int {

View File

@ -1,15 +1,20 @@
package cn.tursom.core.bytebuffer
import cn.tursom.core.logE
import java.nio.ByteBuffer
class DirectNioAdvanceByteBuffer(val buffer: ByteBuffer) : AdvanceByteBuffer {
override val nioBuffer: ByteBuffer get() = buffer
override val readOnly: Boolean get() = buffer.isReadOnly
override var writePosition: Int = buffer.position()
get() = field
var writeMark = 0
override var writePosition: Int
get() {
return if (readMode) writeMark
else buffer.position()
}
set(value) {
if (!readMode) buffer.position(value)
field = value
else buffer.limit(value)
}
override var limit: Int = buffer.limit()
get() = if (!readMode) buffer.limit() else field
@ -34,8 +39,12 @@ class DirectNioAdvanceByteBuffer(val buffer: ByteBuffer) : AdvanceByteBuffer {
override fun readMode() {
if (!readMode) {
writeMark = writePosition
//logE("readMode() $this $writeMark $writePosition ${buffer.position()}")
readMode = true
buffer.flip()
buffer.position(readPosition)
//logE("readMode() $this $writeMark $writePosition ${buffer.position()}")
}
}
@ -43,11 +52,17 @@ class DirectNioAdvanceByteBuffer(val buffer: ByteBuffer) : AdvanceByteBuffer {
if (readMode) {
readMode = false
buffer.limit(capacity)
buffer.position(writePosition)
buffer.position(writeMark)
}
}
override fun clear() {
resumeWriteMode()
buffer.clear()
readPosition = 0
}
override fun toString(): String {
return "DirectNioAdvanceByteBuffer(buffer=$buffer, readMode=$readMode, readPosition=$readPosition, writePosition=$writePosition)"
}
}

View File

@ -122,7 +122,14 @@ class HeapNioAdvanceByteBuffer(val buffer: ByteBuffer) : AdvanceByteBuffer {
override fun getLong() = array.toLong(take(8))
override fun getFloat() = array.toFloat(take(4))
override fun getDouble() = array.toDouble(take(8))
override fun getBytes() = array.copyOfRange(arrayOffset, readAllSize())
override fun getBytes(size: Int): ByteArray {
val readMode = readMode
readMode()
val array = array.copyOfRange(readPosition, useReadSize(size))
if (!readMode) resumeWriteMode(size)
return array
}
override fun getString(size: Int) = String(array, readOffset, useReadSize(size))
override fun writeTo(buffer: ByteArray, bufferOffset: Int, size: Int): Int {
@ -162,4 +169,8 @@ class HeapNioAdvanceByteBuffer(val buffer: ByteBuffer) : AdvanceByteBuffer {
writePosition = writeMark
return slice
}
override fun toString(): String {
return "HeapNioAdvanceByteBuffer(buffer=$buffer, readMode=$_readMode, readMark=$readMark, writeMark=$writeMark, capacity=$capacity, readPosition=$readPosition, size=$size)"
}
}

View File

@ -1,6 +1,8 @@
package cn.tursom.core.bytebuffer
import java.nio.ByteBuffer
import java.util.*
import kotlin.collections.ArrayList
class MultiAdvanceByteBuffer(vararg val buffers: AdvanceByteBuffer) : AdvanceByteBuffer {
init {
@ -87,7 +89,7 @@ class MultiAdvanceByteBuffer(vararg val buffers: AdvanceByteBuffer) : AdvanceByt
override fun getLong(): Long = readBuffer.getLong()
override fun getFloat(): Float = readBuffer.getFloat()
override fun getDouble(): Double = readBuffer.getDouble()
override fun getBytes(): ByteArray = readBuffer.getBytes()
override fun getBytes(size: Int) = readBuffer.getBytes(size)
override fun getString(size: Int): String = readBuffer.getString(size)
override fun put(byte: Byte) = writeBuffer.put(byte)
@ -98,4 +100,9 @@ class MultiAdvanceByteBuffer(vararg val buffers: AdvanceByteBuffer) : AdvanceByt
override fun put(float: Float) = writeBuffer.put(float)
override fun put(double: Double) = writeBuffer.put(double)
override fun put(str: String) = writeBuffer.put(str)
override fun toString(): String {
return "MultiAdvanceByteBuffer(buffers=${Arrays.toString(buffers)}, writeBufferIndex=$writeBufferIndex, readBufferIndex=$readBufferIndex, readMode=$readMode)"
}
}

View File

@ -47,4 +47,8 @@ class DirectMemoryPool(override val blockSize: Int = 1024, override val blockCou
} else {
null
}
override fun toString(): String {
return "DirectMemoryPool(blockSize=$blockSize, blockCount=$blockCount, bitMap=$bitMap)"
}
}

View File

@ -48,4 +48,8 @@ class HeapMemoryPool(override val blockSize: Int = 1024, override val blockCount
} else {
null
}
override fun toString(): String {
return "HeapMemoryPool(blockSize=$blockSize, blockCount=$blockCount, bitMap=$bitMap)"
}
}

View File

@ -23,6 +23,8 @@ interface MemoryPool {
null
}
}
override fun toString(): String
}

View File

@ -22,6 +22,7 @@ class WheelTimer(
init {
thread(isDaemon = true, name = "wheelTimerLooper") {
val startTime = System.currentTimeMillis()
while (!closed) {
position %= wheelSize
@ -44,7 +45,8 @@ class WheelTimer(
}
position++
sleep(tick)
val nextSleep = startTime + tick * position - System.currentTimeMillis()
if (nextSleep > 0) sleep(tick)
}
}
}

View File

@ -80,8 +80,8 @@ class NettyAdvanceByteBuffer(val byteBuf: ByteBuf) : AdvanceByteBuffer {
override fun getFloat(): Float = byteBuf.readFloat()
override fun getDouble(): Double = byteBuf.readDouble()
override fun getBytes(): ByteArray {
val bytes = ByteArray(byteBuf.readableBytes())
override fun getBytes(size: Int): ByteArray {
val bytes = ByteArray(size)
byteBuf.readBytes(bytes)
return bytes
}
@ -140,4 +140,7 @@ class NettyAdvanceByteBuffer(val byteBuf: ByteBuf) : AdvanceByteBuffer {
byteBuf.writeBytes(byteArray, startIndex, endIndex - startIndex)
}
override fun toString(): String {
return "NettyAdvanceByteBuffer(byteBuf=$byteBuf, readMode=$readMode)"
}
}