2021-06-15 19:46:19 +08:00
|
|
|
proc client_idle_sec {name} {
|
|
|
|
set clients [split [r client list] "\r\n"]
|
|
|
|
set c [lsearch -inline $clients *name=$name*]
|
|
|
|
assert {[regexp {idle=([0-9]+)} $c - idle]}
|
|
|
|
return $idle
|
|
|
|
}
|
|
|
|
|
2023-05-04 18:02:08 +08:00
|
|
|
# Calculate query buffer memory of client
|
2021-06-15 19:46:19 +08:00
|
|
|
proc client_query_buffer {name} {
|
|
|
|
set clients [split [r client list] "\r\n"]
|
|
|
|
set c [lsearch -inline $clients *name=$name*]
|
|
|
|
if {[string length $c] > 0} {
|
|
|
|
assert {[regexp {qbuf=([0-9]+)} $c - qbuf]}
|
|
|
|
assert {[regexp {qbuf-free=([0-9]+)} $c - qbuf_free]}
|
|
|
|
return [expr $qbuf + $qbuf_free]
|
|
|
|
}
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
start_server {tags {"querybuf slow"}} {
|
2023-04-16 20:49:26 +08:00
|
|
|
# increase the execution frequency of clientsCron
|
|
|
|
r config set hz 100
|
2023-05-04 18:02:08 +08:00
|
|
|
|
2021-06-15 19:46:19 +08:00
|
|
|
# The test will run at least 2s to check if client query
|
|
|
|
# buffer will be resized when client idle 2s.
|
|
|
|
test "query buffer resized correctly" {
|
2024-05-28 21:09:37 +03:00
|
|
|
|
|
|
|
set rd [valkey_deferring_client]
|
|
|
|
|
2021-06-15 19:46:19 +08:00
|
|
|
$rd client setname test_client
|
2024-05-28 21:09:37 +03:00
|
|
|
$rd read
|
|
|
|
|
|
|
|
# Make sure query buff has size of 0 bytes at start as the client uses the shared qb.
|
|
|
|
assert {[client_query_buffer test_client] == 0}
|
|
|
|
|
2024-06-12 14:27:42 -07:00
|
|
|
# Pause cron to prevent premature shrinking (timing issue).
|
|
|
|
r debug pause-cron 1
|
|
|
|
|
2024-05-28 21:09:37 +03:00
|
|
|
# Send partial command to client to make sure it doesn't use the shared qb.
|
|
|
|
$rd write "*3\r\n\$3\r\nset\r\n\$2\r\na"
|
|
|
|
$rd flush
|
|
|
|
after 100
|
|
|
|
# send the rest of the command
|
|
|
|
$rd write "a\r\n\$1\r\nb\r\n"
|
|
|
|
$rd flush
|
|
|
|
assert_equal {OK} [$rd read]
|
|
|
|
|
2021-06-15 19:46:19 +08:00
|
|
|
set orig_test_client_qbuf [client_query_buffer test_client]
|
2021-08-06 20:50:34 +03:00
|
|
|
# Make sure query buff has less than the peak resize threshold (PROTO_RESIZE_THRESHOLD) 32k
|
|
|
|
# but at least the basic IO reading buffer size (PROTO_IOBUF_LEN) 16k
|
Adjust query buffer resized correctly test to non-jemalloc allocators. (#593)
Test `query buffer resized correctly` start to fail
(https://github.com/valkey-io/valkey/actions/runs/9278013807) with
non-jemalloc allocators after
https://github.com/valkey-io/valkey/pull/258 PR.
With Jemalloc we allocate ~20K for the query buffer, in the test we read
1 byte in the first read, in the second read we make sure we have at
least 16KB free place in the query buffer and we have as Jemalloc
allocated 20KB, But with non jemalloc we allocate in the first read
exactly 16KB. in the second read we check and see that we don't have
16KB free space as we already read 1 byte hence we reallocate this time
greedly (*2 of the requested size of 16KB+1) hence the test condition
that the querybuf size is < 32KB is no longer true
The `query buffer resized correctly test` starts
[failing](https://github.com/valkey-io/valkey/actions/runs/9278013807)
with non-jemalloc allocators after PR #258 .
With jemalloc, we allocate ~20KB for the query buffer. In the test, we
read 1 byte initially and then ensure there is at least 16KB of free
space in the buffer for the second read, which is satisfied by
jemalloc's 20KB allocation. However, with non-jemalloc allocators, the
first read allocates exactly 16KB. When we check again, we don't have
16KB free due to the 1 byte already read. This triggers a greedy
reallocation (doubling the requested size of 16KB+1), causing the query
buffer size to exceed the 32KB limit, thus failing the test condition.
This PR adjusted the test query buffer upper limit to be 32KB +2.
Signed-off-by: Uri Yagelnik <uriy@amazon.com>
2024-06-03 21:15:28 +03:00
|
|
|
set MAX_QUERY_BUFFER_SIZE [expr 32768 + 2] ; # 32k + 2, allowing for potential greedy allocation of (16k + 1) * 2 bytes for the query buffer.
|
|
|
|
assert {$orig_test_client_qbuf >= 16384 && $orig_test_client_qbuf <= $MAX_QUERY_BUFFER_SIZE}
|
2021-06-15 19:46:19 +08:00
|
|
|
|
2024-06-12 14:27:42 -07:00
|
|
|
# Allow shrinking to occur
|
|
|
|
r debug pause-cron 0
|
|
|
|
|
2021-08-06 20:50:34 +03:00
|
|
|
# Check that the initial query buffer is resized after 2 sec
|
2021-06-15 19:46:19 +08:00
|
|
|
wait_for_condition 1000 10 {
|
2024-05-16 09:22:50 +08:00
|
|
|
[client_idle_sec test_client] >= 3 && [client_query_buffer test_client] < $orig_test_client_qbuf
|
2021-06-15 19:46:19 +08:00
|
|
|
} else {
|
2021-08-06 20:50:34 +03:00
|
|
|
fail "query buffer was not resized"
|
2021-06-15 19:46:19 +08:00
|
|
|
}
|
2021-08-06 20:50:34 +03:00
|
|
|
$rd close
|
|
|
|
}
|
2021-06-15 19:46:19 +08:00
|
|
|
|
2021-08-06 20:50:34 +03:00
|
|
|
test "query buffer resized correctly when not idle" {
|
2023-05-04 18:02:08 +08:00
|
|
|
# Pause cron to prevent premature shrinking (timing issue).
|
|
|
|
r debug pause-cron 1
|
|
|
|
|
2021-08-06 20:50:34 +03:00
|
|
|
# Memory will increase by more than 32k due to client query buffer.
|
2024-04-10 10:18:47 -04:00
|
|
|
set rd [valkey_client]
|
2021-08-06 20:50:34 +03:00
|
|
|
$rd client setname test_client
|
2021-06-15 19:46:19 +08:00
|
|
|
|
2021-08-06 20:50:34 +03:00
|
|
|
# Create a large query buffer (more than PROTO_RESIZE_THRESHOLD - 32k)
|
|
|
|
$rd set x [string repeat A 400000]
|
|
|
|
|
|
|
|
# Make sure query buff is larger than the peak resize threshold (PROTO_RESIZE_THRESHOLD) 32k
|
|
|
|
set orig_test_client_qbuf [client_query_buffer test_client]
|
|
|
|
assert {$orig_test_client_qbuf > 32768}
|
|
|
|
|
2023-05-04 18:02:08 +08:00
|
|
|
r debug pause-cron 0
|
|
|
|
|
2021-08-06 20:50:34 +03:00
|
|
|
# Wait for qbuf to shrink due to lower peak
|
|
|
|
set t [clock milliseconds]
|
|
|
|
while true {
|
|
|
|
# Write something smaller, so query buf peak can shrink
|
|
|
|
$rd set x [string repeat A 100]
|
|
|
|
set new_test_client_qbuf [client_query_buffer test_client]
|
|
|
|
if {$new_test_client_qbuf < $orig_test_client_qbuf} { break }
|
|
|
|
if {[expr [clock milliseconds] - $t] > 1000} { break }
|
|
|
|
after 10
|
2021-06-15 19:46:19 +08:00
|
|
|
}
|
2021-08-06 20:50:34 +03:00
|
|
|
# Validate qbuf shrunk but isn't 0 since we maintain room based on latest peak
|
|
|
|
assert {[client_query_buffer test_client] > 0 && [client_query_buffer test_client] < $orig_test_client_qbuf}
|
|
|
|
$rd close
|
2023-05-04 18:02:08 +08:00
|
|
|
} {0} {needs:debug}
|
2023-04-16 20:49:26 +08:00
|
|
|
|
|
|
|
test "query buffer resized correctly with fat argv" {
|
2024-04-10 10:18:47 -04:00
|
|
|
set rd [valkey_client]
|
2023-04-16 20:49:26 +08:00
|
|
|
$rd client setname test_client
|
|
|
|
$rd write "*3\r\n\$3\r\nset\r\n\$1\r\na\r\n\$1000000\r\n"
|
|
|
|
$rd flush
|
|
|
|
|
2024-05-28 21:09:37 +03:00
|
|
|
after 200
|
|
|
|
# Send the start of the arg and make sure the client is not using shared qb for it rather a private buf of > 1000000 size.
|
|
|
|
$rd write "a"
|
|
|
|
$rd flush
|
|
|
|
|
2023-04-16 20:49:26 +08:00
|
|
|
after 20
|
|
|
|
if {[client_query_buffer test_client] < 1000000} {
|
|
|
|
fail "query buffer should not be resized when client idle time smaller than 2s"
|
|
|
|
}
|
|
|
|
|
|
|
|
# Check that the query buffer is resized after 2 sec
|
|
|
|
wait_for_condition 1000 10 {
|
|
|
|
[client_idle_sec test_client] >= 3 && [client_query_buffer test_client] < 1000000
|
|
|
|
} else {
|
|
|
|
fail "query buffer should be resized when client idle time bigger than 2s"
|
|
|
|
}
|
|
|
|
|
|
|
|
$rd close
|
|
|
|
}
|
|
|
|
|
2021-06-15 19:46:19 +08:00
|
|
|
}
|