Add some websocket encode benchmarks. (#1159)
Motivation: To improve the performance of a given component it is very helpful to know how it performs! This patch adds benchmarks and allocation counter tests to NIO for websocket frame encoding. Modifications: - Wrote some benchmarks for WebSocket frame encoding. - Wrote some allocation counter tests for WebSocket frame encoding. - Extend the allocation counter tests to properly account for sub-tests Result: Better insight
This commit is contained in:
Cory Benfield
Johannes Weiss
parent
7e16856aff
commit
37873bb265
|
|
@ -30,24 +30,28 @@ done
|
|||
|
||||
for test in "${all_tests[@]}"; do
|
||||
cat "$tmp/output" # helps debugging
|
||||
total_allocations=$(grep "^test_$test.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
|
||||
not_freed_allocations=$(grep "^test_$test.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
|
||||
max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test"
|
||||
|
||||
info "$test: allocations not freed: $not_freed_allocations"
|
||||
info "$test: total number of mallocs: $total_allocations"
|
||||
while read -r test_case; do
|
||||
test_case=${test_case#test_*}
|
||||
total_allocations=$(grep "^test_$test_case.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
|
||||
not_freed_allocations=$(grep "^test_$test_case.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
|
||||
max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test_case"
|
||||
|
||||
assert_less_than "$not_freed_allocations" 5 # allow some slack
|
||||
assert_greater_than "$not_freed_allocations" -5 # allow some slack
|
||||
if [[ -z "${!max_allowed_env_name+x}" ]]; then
|
||||
info "$test_case: allocations not freed: $not_freed_allocations"
|
||||
info "$test_case: total number of mallocs: $total_allocations"
|
||||
|
||||
assert_less_than "$not_freed_allocations" 5 # allow some slack
|
||||
assert_greater_than "$not_freed_allocations" -5 # allow some slack
|
||||
if [[ -z "${!max_allowed_env_name+x}" ]]; then
|
||||
warn "no reference number of allocations set (set to \$$max_allowed_env_name)"
|
||||
warn "to set current number:"
|
||||
warn " export $max_allowed_env_name=$total_allocations"
|
||||
if [[ -z "${!max_allowed_env_name+x}" ]]; then
|
||||
warn "no reference number of allocations set (set to \$$max_allowed_env_name)"
|
||||
warn "to set current number:"
|
||||
warn " export $max_allowed_env_name=$total_allocations"
|
||||
fi
|
||||
else
|
||||
max_allowed=${!max_allowed_env_name}
|
||||
assert_less_than_or_equal "$total_allocations" "$max_allowed"
|
||||
assert_greater_than "$total_allocations" "$(( max_allowed - 1000))"
|
||||
fi
|
||||
else
|
||||
max_allowed=${!max_allowed_env_name}
|
||||
assert_less_than_or_equal "$total_allocations" "$max_allowed"
|
||||
assert_greater_than "$total_allocations" "$(( max_allowed - 1000))"
|
||||
fi
|
||||
done < <(grep "^test_$test[^\W]*.total_allocations:" "$tmp/output" | cut -d: -f1 | cut -d. -f1 | sort | uniq)
|
||||
done
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ done
|
|||
|
||||
"$here/../../allocation-counter-tests-framework/run-allocation-counter.sh" \
|
||||
-p "$here/../../.." \
|
||||
-m NIO -m NIOHTTP1 \
|
||||
-m NIO -m NIOHTTP1 -m NIOWebSocket \
|
||||
-s "$here/shared.swift" \
|
||||
-t "$tmp_dir" \
|
||||
"$here"/test_*.swift
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the SwiftNIO open source project
|
||||
//
|
||||
// Copyright (c) 2017-2019 Apple Inc. and the SwiftNIO project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import NIO
|
||||
import NIOWebSocket
|
||||
|
||||
func doSendFramesHoldingBuffer(channel: EmbeddedChannel, number numberOfFrameSends: Int, data originalData: [UInt8], spareBytesAtFront: Int) throws -> Int {
|
||||
var data = channel.allocator.buffer(capacity: originalData.count + spareBytesAtFront)
|
||||
data.moveWriterIndex(forwardBy: spareBytesAtFront)
|
||||
data.moveReaderIndex(forwardBy: spareBytesAtFront)
|
||||
data.writeBytes(originalData)
|
||||
|
||||
let frame = WebSocketFrame(opcode: .binary, data: data, extensionData: nil)
|
||||
|
||||
// We're interested in counting allocations, so this test reads the data from the EmbeddedChannel
|
||||
// to force the data out of memory.
|
||||
for _ in 0..<numberOfFrameSends {
|
||||
channel.writeAndFlush(frame, promise: nil)
|
||||
_ = try channel.readOutbound(as: ByteBuffer.self)
|
||||
_ = try channel.readOutbound(as: ByteBuffer.self)
|
||||
}
|
||||
|
||||
return numberOfFrameSends
|
||||
}
|
||||
|
||||
|
||||
func doSendFramesNewBuffer(channel: EmbeddedChannel, number numberOfFrameSends: Int, data originalData: [UInt8], spareBytesAtFront: Int) throws -> Int {
|
||||
for _ in 0..<numberOfFrameSends {
|
||||
// We need a new allocation every time to drop the original data ref.
|
||||
var data = channel.allocator.buffer(capacity: originalData.count + spareBytesAtFront)
|
||||
data.moveWriterIndex(forwardBy: spareBytesAtFront)
|
||||
data.moveReaderIndex(forwardBy: spareBytesAtFront)
|
||||
data.writeBytes(originalData)
|
||||
let frame = WebSocketFrame(opcode: .binary, data: data, extensionData: nil)
|
||||
|
||||
// We're interested in counting allocations, so this test reads the data from the EmbeddedChannel
|
||||
// to force the data out of memory.
|
||||
channel.writeAndFlush(frame, promise: nil)
|
||||
_ = try channel.readOutbound(as: ByteBuffer.self)
|
||||
_ = try channel.readOutbound(as: ByteBuffer.self)
|
||||
}
|
||||
|
||||
return numberOfFrameSends
|
||||
}
|
||||
|
||||
|
||||
func run(identifier: String) {
|
||||
let channel = EmbeddedChannel()
|
||||
try! channel.pipeline.addHandler(WebSocketFrameEncoder()).wait()
|
||||
let data = Array(repeating: UInt8(0), count: 1024)
|
||||
|
||||
measure(identifier: identifier + "_holding_buffer") {
|
||||
let numberDone = try! doSendFramesHoldingBuffer(channel: channel, number: 1000, data: data, spareBytesAtFront: 0)
|
||||
precondition(numberDone == 1000)
|
||||
return numberDone
|
||||
}
|
||||
|
||||
measure(identifier: identifier + "_holding_buffer_with_space") {
|
||||
let numberDone = try! doSendFramesHoldingBuffer(channel: channel, number: 1000, data: data, spareBytesAtFront: 8)
|
||||
precondition(numberDone == 1000)
|
||||
return numberDone
|
||||
}
|
||||
|
||||
measure(identifier: identifier + "_new_buffer") {
|
||||
let numberDone = try! doSendFramesNewBuffer(channel: channel, number: 1000, data: data, spareBytesAtFront: 0)
|
||||
precondition(numberDone == 1000)
|
||||
return numberDone
|
||||
}
|
||||
|
||||
measure(identifier: identifier + "_new_buffer_with_space") {
|
||||
let numberDone = try! doSendFramesNewBuffer(channel: channel, number: 1000, data: data, spareBytesAtFront: 8)
|
||||
precondition(numberDone == 1000)
|
||||
return numberDone
|
||||
}
|
||||
|
||||
_ = try! channel.finish()
|
||||
}
|
||||
|
|
@ -54,7 +54,7 @@ var targets: [PackageDescription.Target] = [
|
|||
.target(name: "NIOWebSocketClient",
|
||||
dependencies: ["NIO", "NIOHTTP1", "NIOWebSocket"]),
|
||||
.target(name: "NIOPerformanceTester",
|
||||
dependencies: ["NIO", "NIOHTTP1", "NIOFoundationCompat"]),
|
||||
dependencies: ["NIO", "NIOHTTP1", "NIOFoundationCompat", "NIOWebSocket"]),
|
||||
.target(name: "NIOMulticastChat",
|
||||
dependencies: ["NIO"]),
|
||||
.target(name: "NIOUDPEchoServer",
|
||||
|
|
|
|||
|
|
@ -237,7 +237,7 @@ class EmbeddedChannelCore: ChannelCore {
|
|||
|
||||
func flush0() {
|
||||
let pendings = self.pendingOutboundBuffer
|
||||
self.pendingOutboundBuffer.removeAll()
|
||||
self.pendingOutboundBuffer.removeAll(keepingCapacity: true)
|
||||
for dataAndPromise in pendings {
|
||||
self.addToBuffer(buffer: &self.outboundBuffer, data: dataAndPromise.0)
|
||||
dataAndPromise.1?.succeed(())
|
||||
|
|
|
|||
|
|
@ -13,14 +13,12 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
|
||||
protocol Benchmark: class {
|
||||
init()
|
||||
func setUp() throws
|
||||
func tearDown()
|
||||
func run() throws -> Int
|
||||
}
|
||||
|
||||
func measureAndPrint<B: Benchmark>(desc: String, benchmark: B.Type) throws {
|
||||
let bench = B()
|
||||
func measureAndPrint<B: Benchmark>(desc: String, benchmark bench: B) throws {
|
||||
try bench.setUp()
|
||||
defer {
|
||||
bench.tearDown()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the SwiftNIO open source project
|
||||
//
|
||||
// Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import NIO
|
||||
import NIOWebSocket
|
||||
|
||||
final class WebSocketFrameEncoderBenchmark {
|
||||
private let channel: EmbeddedChannel
|
||||
private let dataSize: Int
|
||||
private let data: ByteBuffer
|
||||
private let runCount: Int
|
||||
private let dataStrategy: DataStrategy
|
||||
private let cowStrategy: CoWStrategy
|
||||
private var frame: WebSocketFrame?
|
||||
|
||||
init(dataSize: Int, runCount: Int, dataStrategy: DataStrategy, cowStrategy: CoWStrategy) {
|
||||
self.channel = EmbeddedChannel()
|
||||
self.dataSize = dataSize
|
||||
self.runCount = runCount
|
||||
self.dataStrategy = dataStrategy
|
||||
self.cowStrategy = cowStrategy
|
||||
self.data = ByteBufferAllocator().buffer(size: dataSize, dataStrategy: dataStrategy)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
extension WebSocketFrameEncoderBenchmark {
|
||||
enum DataStrategy {
|
||||
case spaceAtFront
|
||||
case noSpaceAtFront
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
extension WebSocketFrameEncoderBenchmark {
|
||||
enum CoWStrategy {
|
||||
case always
|
||||
case never
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
extension WebSocketFrameEncoderBenchmark: Benchmark {
|
||||
func setUp() throws {
|
||||
// We want the pipeline walk to have some cost.
|
||||
for _ in 0..<3 {
|
||||
try! self.channel.pipeline.addHandler(NoOpOutboundHandler()).wait()
|
||||
}
|
||||
try! self.channel.pipeline.addHandler(WebSocketFrameEncoder()).wait()
|
||||
self.frame = WebSocketFrame(opcode: .binary, data: self.data, extensionData: nil)
|
||||
}
|
||||
|
||||
func tearDown() {
|
||||
_ = try! self.channel.finish()
|
||||
}
|
||||
|
||||
func run() throws -> Int {
|
||||
switch self.cowStrategy {
|
||||
case .always:
|
||||
let frame = self.frame!
|
||||
return self.runWithCoWs(frame: frame)
|
||||
case .never:
|
||||
return self.runWithoutCoWs()
|
||||
}
|
||||
}
|
||||
|
||||
private func runWithCoWs(frame: WebSocketFrame) -> Int {
|
||||
for _ in 0..<self.runCount {
|
||||
self.channel.write(frame, promise: nil)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
private func runWithoutCoWs() -> Int {
|
||||
for _ in 0..<self.runCount {
|
||||
// To avoid CoWs this has to be a new buffer every time. This is expensive, sadly, so tests using this strategy
|
||||
// must do fewer iterations.
|
||||
let data = self.channel.allocator.buffer(size: self.dataSize, dataStrategy: self.dataStrategy)
|
||||
let frame = WebSocketFrame(opcode: .binary, data: data, extensionData: nil)
|
||||
self.channel.write(frame, promise: nil)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
extension ByteBufferAllocator {
|
||||
fileprivate func buffer(size: Int, dataStrategy: WebSocketFrameEncoderBenchmark.DataStrategy) -> ByteBuffer {
|
||||
var data: ByteBuffer
|
||||
|
||||
switch dataStrategy {
|
||||
case .noSpaceAtFront:
|
||||
data = self.buffer(capacity: size)
|
||||
case .spaceAtFront:
|
||||
data = self.buffer(capacity: size + 16)
|
||||
data.moveWriterIndex(forwardBy: 16)
|
||||
data.moveReaderIndex(forwardBy: 16)
|
||||
}
|
||||
|
||||
data.writeBytes(repeatElement(0, count: size))
|
||||
return data
|
||||
}
|
||||
}
|
||||
|
||||
fileprivate final class NoOpOutboundHandler: ChannelOutboundHandler {
|
||||
typealias OutboundIn = Any
|
||||
typealias OutboundOut = Any
|
||||
}
|
||||
|
|
@ -714,5 +714,28 @@ measureAndPrint(desc: "future_reduce_into_10k_futures") {
|
|||
return try! EventLoopFuture<Int>.reduce(into: 0, oneHundredFutures, on: el1, { $0 += $1 }).wait()
|
||||
}
|
||||
|
||||
try measureAndPrint(desc: "channel_pipeline_1m_events", benchmark: ChannelPipelineBenchmark())
|
||||
|
||||
try measureAndPrint(desc: "channel_pipeline_1m_events", benchmark: ChannelPipelineBenchmark.self)
|
||||
try measureAndPrint(desc: "websocket_encode_50b_space_at_front_1m_frames_cow",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 50, runCount: 1_000_000, dataStrategy: .spaceAtFront, cowStrategy: .always))
|
||||
|
||||
try measureAndPrint(desc: "websocket_encode_1kb_space_at_front_100k_frames_cow",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 1024, runCount: 100_000, dataStrategy: .spaceAtFront, cowStrategy: .always))
|
||||
|
||||
try measureAndPrint(desc: "websocket_encode_50b_no_space_at_front_1m_frames_cow",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 50, runCount: 1_000_000, dataStrategy: .noSpaceAtFront, cowStrategy: .always))
|
||||
|
||||
try measureAndPrint(desc: "websocket_encode_1kb_no_space_at_front_100k_frames_cow",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 1024, runCount: 100_000, dataStrategy: .noSpaceAtFront, cowStrategy: .always))
|
||||
|
||||
try measureAndPrint(desc: "websocket_encode_50b_space_at_front_10k_frames",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 50, runCount: 10_000, dataStrategy: .spaceAtFront, cowStrategy: .never))
|
||||
|
||||
try measureAndPrint(desc: "websocket_encode_1kb_space_at_front_1k_frames",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 1024, runCount: 1_000, dataStrategy: .spaceAtFront, cowStrategy: .never))
|
||||
|
||||
try measureAndPrint(desc: "websocket_encode_50b_no_space_at_front_10k_frames",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 50, runCount: 10_000, dataStrategy: .noSpaceAtFront, cowStrategy: .never))
|
||||
|
||||
try measureAndPrint(desc: "websocket_encode_1kb_no_space_at_front_1k_frames",
|
||||
benchmark: WebSocketFrameEncoderBenchmark(dataSize: 1024, runCount: 1_000, dataStrategy: .noSpaceAtFront, cowStrategy: .never))
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ services:
|
|||
- MAX_ALLOCS_ALLOWED_creating_10000_headers=10100
|
||||
- MAX_ALLOCS_ALLOWED_scheduling_10000_executions=20150
|
||||
- MAX_ALLOCS_ALLOWED_modifying_1000_circular_buffer_elements=50
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_holding_buffer=4010
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_holding_buffer_with_space=4010
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_new_buffer=6010
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_new_buffer_with_space=6010
|
||||
- SANITIZER_ARG=--sanitize=thread
|
||||
|
||||
performance-test:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ services:
|
|||
- MAX_ALLOCS_ALLOWED_scheduling_10000_executions=20150
|
||||
- MAX_ALLOCS_ALLOWED_creating_10000_headers=10100
|
||||
- MAX_ALLOCS_ALLOWED_modifying_1000_circular_buffer_elements=50
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_holding_buffer=4010
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_holding_buffer_with_space=4010
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_new_buffer=6010
|
||||
- MAX_ALLOCS_ALLOWED_encode_1000_ws_frames_new_buffer_with_space=6010
|
||||
|
||||
performance-test:
|
||||
image: swift-nio:18.04-5.0
|
||||
|
|
|
|||
Loading…
Reference in New Issue