2021-09-13 15:39:11 +08:00
|
|
|
start_server {tags {"psync2 external:skip"}} {
|
|
|
|
|
start_server {} {
|
|
|
|
|
start_server {} {
|
|
|
|
|
set master [srv 0 client]
|
|
|
|
|
set master_host [srv 0 host]
|
|
|
|
|
set master_port [srv 0 port]
|
|
|
|
|
|
|
|
|
|
set replica [srv -1 client]
|
|
|
|
|
set replica_host [srv -1 host]
|
|
|
|
|
set replica_port [srv -1 port]
|
|
|
|
|
|
|
|
|
|
set sub_replica [srv -2 client]
|
|
|
|
|
|
2023-04-18 21:14:26 +08:00
|
|
|
# Make sure the server saves an RDB on shutdown
|
|
|
|
|
$master config set save "3600 1"
|
|
|
|
|
|
2023-08-16 08:36:40 +03:00
|
|
|
# Because we will test partial resync later, we don't want a timeout to cause
|
2022-12-22 20:23:14 +08:00
|
|
|
# the master-replica disconnect, then the extra reconnections will break the
|
|
|
|
|
# sync_partial_ok stat test
|
|
|
|
|
$master config set repl-timeout 3600
|
|
|
|
|
$replica config set repl-timeout 3600
|
|
|
|
|
$sub_replica config set repl-timeout 3600
|
|
|
|
|
|
2023-05-06 11:53:28 +08:00
|
|
|
# Avoid PINGs
|
|
|
|
|
$master config set repl-ping-replica-period 3600
|
|
|
|
|
$master config rewrite
|
|
|
|
|
|
2021-09-13 15:39:11 +08:00
|
|
|
# Build replication chain
|
|
|
|
|
$replica replicaof $master_host $master_port
|
|
|
|
|
$sub_replica replicaof $replica_host $replica_port
|
|
|
|
|
|
|
|
|
|
wait_for_condition 50 100 {
|
|
|
|
|
[status $replica master_link_status] eq {up} &&
|
|
|
|
|
[status $sub_replica master_link_status] eq {up}
|
|
|
|
|
} else {
|
|
|
|
|
fail "Replication not started."
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-06 11:53:28 +08:00
|
|
|
test "PSYNC2: Partial resync after Master restart using RDB aux fields when offset is 0" {
|
|
|
|
|
assert {[status $master master_repl_offset] == 0}
|
|
|
|
|
|
|
|
|
|
set replid [status $master master_replid]
|
|
|
|
|
$replica config resetstat
|
|
|
|
|
|
|
|
|
|
catch {
|
|
|
|
|
restart_server 0 true false true now
|
|
|
|
|
set master [srv 0 client]
|
|
|
|
|
}
|
|
|
|
|
wait_for_condition 50 1000 {
|
|
|
|
|
[status $replica master_link_status] eq {up} &&
|
|
|
|
|
[status $sub_replica master_link_status] eq {up}
|
|
|
|
|
} else {
|
|
|
|
|
fail "Replicas didn't sync after master restart"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Make sure master restore replication info correctly
|
|
|
|
|
assert {[status $master master_replid] != $replid}
|
|
|
|
|
assert {[status $master master_repl_offset] == 0}
|
|
|
|
|
assert {[status $master master_replid2] eq $replid}
|
|
|
|
|
assert {[status $master second_repl_offset] == 1}
|
|
|
|
|
|
|
|
|
|
# Make sure master set replication backlog correctly
|
|
|
|
|
assert {[status $master repl_backlog_active] == 1}
|
|
|
|
|
assert {[status $master repl_backlog_first_byte_offset] == 1}
|
|
|
|
|
assert {[status $master repl_backlog_histlen] == 0}
|
|
|
|
|
|
|
|
|
|
# Partial resync after Master restart
|
|
|
|
|
assert {[status $master sync_partial_ok] == 1}
|
|
|
|
|
assert {[status $replica sync_partial_ok] == 1}
|
|
|
|
|
}
|
2021-09-13 15:39:11 +08:00
|
|
|
|
|
|
|
|
# Generate some data
|
|
|
|
|
createComplexDataset $master 1000
|
|
|
|
|
|
2023-05-06 11:53:28 +08:00
|
|
|
test "PSYNC2: Partial resync after Master restart using RDB aux fields with data" {
|
2021-09-13 15:39:11 +08:00
|
|
|
wait_for_condition 500 100 {
|
|
|
|
|
[status $master master_repl_offset] == [status $replica master_repl_offset] &&
|
|
|
|
|
[status $master master_repl_offset] == [status $sub_replica master_repl_offset]
|
|
|
|
|
} else {
|
|
|
|
|
fail "Replicas and master offsets were unable to match *exactly*."
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
set replid [status $master master_replid]
|
|
|
|
|
set offset [status $master master_repl_offset]
|
|
|
|
|
$replica config resetstat
|
|
|
|
|
|
|
|
|
|
catch {
|
2022-01-02 08:50:15 +01:00
|
|
|
# SHUTDOWN NOW ensures master doesn't send GETACK to replicas before
|
|
|
|
|
# shutting down which would affect the replication offset.
|
|
|
|
|
restart_server 0 true false true now
|
2021-09-13 15:39:11 +08:00
|
|
|
set master [srv 0 client]
|
|
|
|
|
}
|
|
|
|
|
wait_for_condition 50 1000 {
|
|
|
|
|
[status $replica master_link_status] eq {up} &&
|
|
|
|
|
[status $sub_replica master_link_status] eq {up}
|
|
|
|
|
} else {
|
|
|
|
|
fail "Replicas didn't sync after master restart"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Make sure master restore replication info correctly
|
|
|
|
|
assert {[status $master master_replid] != $replid}
|
|
|
|
|
assert {[status $master master_repl_offset] == $offset}
|
|
|
|
|
assert {[status $master master_replid2] eq $replid}
|
|
|
|
|
assert {[status $master second_repl_offset] == [expr $offset+1]}
|
|
|
|
|
|
|
|
|
|
# Make sure master set replication backlog correctly
|
|
|
|
|
assert {[status $master repl_backlog_active] == 1}
|
|
|
|
|
assert {[status $master repl_backlog_first_byte_offset] == [expr $offset+1]}
|
|
|
|
|
assert {[status $master repl_backlog_histlen] == 0}
|
|
|
|
|
|
|
|
|
|
# Partial resync after Master restart
|
|
|
|
|
assert {[status $master sync_partial_ok] == 1}
|
|
|
|
|
assert {[status $replica sync_partial_ok] == 1}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
test "PSYNC2: Partial resync after Master restart using RDB aux fields with expire" {
|
|
|
|
|
$master debug set-active-expire 0
|
|
|
|
|
for {set j 0} {$j < 1024} {incr j} {
|
|
|
|
|
$master select [expr $j%16]
|
|
|
|
|
$master set $j somevalue px 10
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
after 20
|
|
|
|
|
|
2022-01-02 08:50:15 +01:00
|
|
|
# Wait until master has received ACK from replica. If the master thinks
|
|
|
|
|
# that any replica is lagging when it shuts down, master would send
|
|
|
|
|
# GETACK to the replicas, affecting the replication offset.
|
|
|
|
|
set offset [status $master master_repl_offset]
|
2021-09-13 15:39:11 +08:00
|
|
|
wait_for_condition 500 100 {
|
2022-01-02 08:50:15 +01:00
|
|
|
[string match "*slave0:*,offset=$offset,*" [$master info replication]] &&
|
|
|
|
|
$offset == [status $replica master_repl_offset] &&
|
|
|
|
|
$offset == [status $sub_replica master_repl_offset]
|
2021-09-13 15:39:11 +08:00
|
|
|
} else {
|
|
|
|
|
show_cluster_status
|
|
|
|
|
fail "Replicas and master offsets were unable to match *exactly*."
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
set offset [status $master master_repl_offset]
|
|
|
|
|
$replica config resetstat
|
|
|
|
|
|
|
|
|
|
catch {
|
2022-01-02 08:50:15 +01:00
|
|
|
# Unlike the test above, here we use SIGTERM, which behaves
|
|
|
|
|
# differently compared to SHUTDOWN NOW if there are lagging
|
|
|
|
|
# replicas. This is just to increase coverage and let each test use
|
|
|
|
|
# a different shutdown approach. In this case there are no lagging
|
|
|
|
|
# replicas though.
|
2021-09-13 15:39:11 +08:00
|
|
|
restart_server 0 true false
|
|
|
|
|
set master [srv 0 client]
|
|
|
|
|
}
|
|
|
|
|
wait_for_condition 50 1000 {
|
|
|
|
|
[status $replica master_link_status] eq {up} &&
|
|
|
|
|
[status $sub_replica master_link_status] eq {up}
|
|
|
|
|
} else {
|
|
|
|
|
fail "Replicas didn't sync after master restart"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
set expired_offset [status $master repl_backlog_histlen]
|
|
|
|
|
# Stale keys expired and master_repl_offset grows correctly
|
|
|
|
|
assert {[status $master rdb_last_load_keys_expired] == 1024}
|
|
|
|
|
assert {[status $master master_repl_offset] == [expr $offset+$expired_offset]}
|
|
|
|
|
|
|
|
|
|
# Partial resync after Master restart
|
|
|
|
|
assert {[status $master sync_partial_ok] == 1}
|
|
|
|
|
assert {[status $replica sync_partial_ok] == 1}
|
|
|
|
|
|
|
|
|
|
set digest [$master debug digest]
|
|
|
|
|
assert {$digest eq [$replica debug digest]}
|
|
|
|
|
assert {$digest eq [$sub_replica debug digest]}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
test "PSYNC2: Full resync after Master restart when too many key expired" {
|
|
|
|
|
$master config set repl-backlog-size 16384
|
|
|
|
|
$master config rewrite
|
|
|
|
|
|
|
|
|
|
$master debug set-active-expire 0
|
Replication backlog and replicas use one global shared replication buffer (#9166)
## Background
For redis master, one replica uses one copy of replication buffer, that is a big waste of memory,
more replicas more waste, and allocate/free memory for every reply list also cost much.
If we set client-output-buffer-limit small and write traffic is heavy, master may disconnect with
replicas and can't finish synchronization with replica. If we set client-output-buffer-limit big,
master may be OOM when there are many replicas that separately keep much memory.
Because replication buffers of different replica client are the same, one simple idea is that
all replicas only use one replication buffer, that will effectively save memory.
Since replication backlog content is the same as replicas' output buffer, now we
can discard replication backlog memory and use global shared replication buffer
to implement replication backlog mechanism.
## Implementation
I create one global "replication buffer" which contains content of replication stream.
The structure of "replication buffer" is similar to the reply list that exists in every client.
But the node of list is `replBufBlock`, which has `id, repl_offset, refcount` fields.
```c
/* Replication buffer blocks is the list of replBufBlock.
*
* +--------------+ +--------------+ +--------------+
* | refcount = 1 | ... | refcount = 0 | ... | refcount = 2 |
* +--------------+ +--------------+ +--------------+
* | / \
* | / \
* | / \
* Repl Backlog Replia_A Replia_B
*
* Each replica or replication backlog increments only the refcount of the
* 'ref_repl_buf_node' which it points to. So when replica walks to the next
* node, it should first increase the next node's refcount, and when we trim
* the replication buffer nodes, we remove node always from the head node which
* refcount is 0. If the refcount of the head node is not 0, we must stop
* trimming and never iterate the next node. */
/* Similar with 'clientReplyBlock', it is used for shared buffers between
* all replica clients and replication backlog. */
typedef struct replBufBlock {
int refcount; /* Number of replicas or repl backlog using. */
long long id; /* The unique incremental number. */
long long repl_offset; /* Start replication offset of the block. */
size_t size, used;
char buf[];
} replBufBlock;
```
So now when we feed replication stream into replication backlog and all replicas, we only need
to feed stream into replication buffer `feedReplicationBuffer`. In this function, we set some fields of
replication backlog and replicas to references of the global replication buffer blocks. And we also
need to check replicas' output buffer limit to free if exceeding `client-output-buffer-limit`, and trim
replication backlog if exceeding `repl-backlog-size`.
When sending reply to replicas, we also need to iterate replication buffer blocks and send its
content, when totally sending one block for replica, we decrease current node count and
increase the next current node count, and then free the block which reference is 0 from the
head of replication buffer blocks.
Since now we use linked list to manage replication backlog, it may cost much time for iterating
all linked list nodes to find corresponding replication buffer node. So we create a rax tree to
store some nodes for index, but to avoid rax tree occupying too much memory, i record
one per 64 nodes for index.
Currently, to make partial resynchronization as possible as much, we always let replication
backlog as the last reference of replication buffer blocks, backlog size may exceeds our setting
if slow replicas that reference vast replication buffer blocks, and this method doesn't increase
memory usage since they share replication buffer. To avoid freezing server for freeing unreferenced
replication buffer blocks when we need to trim backlog for exceeding backlog size setting,
we trim backlog incrementally (free 64 blocks per call now), and make it faster in
`beforeSleep` (free 640 blocks).
### Other changes
- `mem_total_replication_buffers`: we add this field in INFO command, it means the total
memory of replication buffers used.
- `mem_clients_slaves`: now even replica is slow to replicate, and its output buffer memory
is not 0, but it still may be 0, since replication backlog and replicas share one global replication
buffer, only if replication buffer memory is more than the repl backlog setting size, we consider
the excess as replicas' memory. Otherwise, we think replication buffer memory is the consumption
of repl backlog.
- Key eviction
Since all replicas and replication backlog share global replication buffer, we think only the
part of exceeding backlog size the extra separate consumption of replicas.
Because we trim backlog incrementally in the background, backlog size may exceeds our
setting if slow replicas that reference vast replication buffer blocks disconnect.
To avoid massive eviction loop, we don't count the delayed freed replication backlog into
used memory even if there are no replicas, i.e. we also regard this memory as replicas's memory.
- `client-output-buffer-limit` check for replica clients
It doesn't make sense to set the replica clients output buffer limit lower than the repl-backlog-size
config (partial sync will succeed and then replica will get disconnected). Such a configuration is
ignored (the size of repl-backlog-size will be used). This doesn't have memory consumption
implications since the replica client will share the backlog buffers memory.
- Drop replication backlog after loading data if needed
We always create replication backlog if server is a master, we need it because we put DELs in
it when loading expired keys in RDB, but if RDB doesn't have replication info or there is no rdb,
it is not possible to support partial resynchronization, to avoid extra memory of replication backlog,
we drop it.
- Multi IO threads
Since all replicas and replication backlog use global replication buffer, if I/O threads are enabled,
to guarantee data accessing thread safe, we must let main thread handle sending the output buffer
to all replicas. But before, other IO threads could handle sending output buffer of all replicas.
## Other optimizations
This solution resolve some other problem:
- When replicas disconnect with master since of out of output buffer limit, releasing the output
buffer of replicas may freeze server if we set big `client-output-buffer-limit` for replicas, but now,
it doesn't cause freezing.
- This implementation may mitigate reply list copy cost time(also freezes server) when one replication
has huge reply buffer and another replica can copy buffer for full synchronization. now, we just copy
reference info, it is very light.
- If we set replication backlog size big, it also may cost much time to copy replication backlog into
replica's output buffer. But this commit eliminates this problem.
- Resizing replication backlog size doesn't empty current replication backlog content.
2021-10-25 14:24:31 +08:00
|
|
|
# Make sure replication backlog is full and will be trimmed.
|
|
|
|
|
for {set j 0} {$j < 2048} {incr j} {
|
2021-09-13 15:39:11 +08:00
|
|
|
$master select [expr $j%16]
|
|
|
|
|
$master set $j somevalue px 10
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
after 20
|
|
|
|
|
|
|
|
|
|
wait_for_condition 500 100 {
|
|
|
|
|
[status $master master_repl_offset] == [status $replica master_repl_offset] &&
|
|
|
|
|
[status $master master_repl_offset] == [status $sub_replica master_repl_offset]
|
|
|
|
|
} else {
|
|
|
|
|
fail "Replicas and master offsets were unable to match *exactly*."
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$replica config resetstat
|
|
|
|
|
|
|
|
|
|
catch {
|
2022-01-02 08:50:15 +01:00
|
|
|
# Unlike the test above, here we use SIGTERM. This is just to
|
|
|
|
|
# increase coverage and let each test use a different shutdown
|
|
|
|
|
# approach.
|
2021-09-13 15:39:11 +08:00
|
|
|
restart_server 0 true false
|
|
|
|
|
set master [srv 0 client]
|
|
|
|
|
}
|
|
|
|
|
wait_for_condition 50 1000 {
|
|
|
|
|
[status $replica master_link_status] eq {up} &&
|
|
|
|
|
[status $sub_replica master_link_status] eq {up}
|
|
|
|
|
} else {
|
|
|
|
|
fail "Replicas didn't sync after master restart"
|
|
|
|
|
}
|
Dual channel replication (#60)
In this PR we introduce the main benefit of dual channel replication by
continuously steaming the COB (client output buffers) in parallel to the
RDB and thus keeping the primary's side COB small AND accelerating the
overall sync process. By streaming the replication data to the replica
during the full sync, we reduce
1. Memory load from the primary's node.
2. CPU load from the primary's main process. [Latest performance
tests](#data)
## Motivation
* Reduce primary memory load. We do that by moving the COB tracking to
the replica side. This also decrease the chance for COB overruns. Note
that primary's input buffer limits at the replica side are less
restricted then primary's COB as the replica plays less critical part in
the replication group. While increasing the primary’s COB may end up
with primary reaching swap and clients suffering, at replica side we’re
more at ease with it. Larger COB means better chance to sync
successfully.
* Reduce primary main process CPU load. By opening a new, dedicated
connection for the RDB transfer, child processes can have direct access
to the new connection. Due to TLS connection restrictions, this was not
possible using one main connection. We eliminate the need for the child
process to use the primary's child-proc -> main-proc pipeline, thus
freeing up the main process to process clients queries.
## Dual Channel Replication high level interface design
- Dual channel replication begins when the replica sends a `REPLCONF
CAPA DUALCHANNEL` to the primary during initial
handshake. This is used to state that the replica is capable of dual
channel sync and that this is the replica's main channel, which is not
used for snapshot transfer.
- When replica lacks sufficient data for PSYNC, the primary will send
`-FULLSYNCNEEDED` response instead
of RDB data. As a next step, the replica creates a new connection
(rdb-channel) and configures it against
the primary with the appropriate capabilities and requirements. The
replica then requests a sync
using the RDB channel.
- Prior to forking, the primary sends the replica the snapshot's end
repl-offset, and attaches the replica
to the replication backlog to keep repl data until the replica requests
psync. The replica uses the main
channel to request a PSYNC starting at the snapshot end offset.
- The primary main threads sends incremental changes via the main
channel, while the bgsave process
sends the RDB directly to the replica via the rdb-channel. As for the
replica, the incremental
changes are stored on a local buffer, while the RDB is loaded into
memory.
- Once the replica completes loading the rdb, it drops the
rdb-connection and streams the accumulated incremental
changes into memory. Repl steady state continues normally.
## New replica state machine
## Data <a name="data"></a>
## Explanation
These graphs demonstrate performance improvements during full sync
sessions using rdb-channel + streaming rdb directly from the background
process to the replica.
First graph- with at most 50 clients and light weight commands, we saw
5%-7.5% improvement in write latency during sync session.
Two graphs below- full sync was tested during heavy read commands from
the primary (such as sdiff, sunion on large sets). In that case, the
child process writes to the replica without sharing CPU with the loaded
main process. As a result, this not only improves client response time,
but may also shorten sync time by about 50%. The shorter sync time
results in less memory being used to store replication diffs (>60% in
some of the tested cases).
## Test setup
Both primary and replica in the performance tests ran on the same
machine. RDB size in all tests is 3.7gb. I generated write load using
valkey-benchmark ` ./valkey-benchmark -r 100000 -n 6000000 lpush my_list
__rand_int__`.
---------
Signed-off-by: naglera <anagler123@gmail.com>
Signed-off-by: naglera <58042354+naglera@users.noreply.github.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Ping Xie <pingxie@outlook.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-07-17 23:59:33 +03:00
|
|
|
set dualchannel [lindex [r config get dual-channel-replication-enabled] 1]
|
|
|
|
|
set psync_count 0
|
|
|
|
|
if {$dualchannel == "yes"} {
|
|
|
|
|
# Expect one fake psync
|
|
|
|
|
set psync_count 1
|
|
|
|
|
}
|
2021-09-13 15:39:11 +08:00
|
|
|
|
|
|
|
|
# Replication backlog is full
|
|
|
|
|
assert {[status $master repl_backlog_first_byte_offset] > [status $master second_repl_offset]}
|
Dual channel replication (#60)
In this PR we introduce the main benefit of dual channel replication by
continuously steaming the COB (client output buffers) in parallel to the
RDB and thus keeping the primary's side COB small AND accelerating the
overall sync process. By streaming the replication data to the replica
during the full sync, we reduce
1. Memory load from the primary's node.
2. CPU load from the primary's main process. [Latest performance
tests](#data)
## Motivation
* Reduce primary memory load. We do that by moving the COB tracking to
the replica side. This also decrease the chance for COB overruns. Note
that primary's input buffer limits at the replica side are less
restricted then primary's COB as the replica plays less critical part in
the replication group. While increasing the primary’s COB may end up
with primary reaching swap and clients suffering, at replica side we’re
more at ease with it. Larger COB means better chance to sync
successfully.
* Reduce primary main process CPU load. By opening a new, dedicated
connection for the RDB transfer, child processes can have direct access
to the new connection. Due to TLS connection restrictions, this was not
possible using one main connection. We eliminate the need for the child
process to use the primary's child-proc -> main-proc pipeline, thus
freeing up the main process to process clients queries.
## Dual Channel Replication high level interface design
- Dual channel replication begins when the replica sends a `REPLCONF
CAPA DUALCHANNEL` to the primary during initial
handshake. This is used to state that the replica is capable of dual
channel sync and that this is the replica's main channel, which is not
used for snapshot transfer.
- When replica lacks sufficient data for PSYNC, the primary will send
`-FULLSYNCNEEDED` response instead
of RDB data. As a next step, the replica creates a new connection
(rdb-channel) and configures it against
the primary with the appropriate capabilities and requirements. The
replica then requests a sync
using the RDB channel.
- Prior to forking, the primary sends the replica the snapshot's end
repl-offset, and attaches the replica
to the replication backlog to keep repl data until the replica requests
psync. The replica uses the main
channel to request a PSYNC starting at the snapshot end offset.
- The primary main threads sends incremental changes via the main
channel, while the bgsave process
sends the RDB directly to the replica via the rdb-channel. As for the
replica, the incremental
changes are stored on a local buffer, while the RDB is loaded into
memory.
- Once the replica completes loading the rdb, it drops the
rdb-connection and streams the accumulated incremental
changes into memory. Repl steady state continues normally.
## New replica state machine
## Data <a name="data"></a>
## Explanation
These graphs demonstrate performance improvements during full sync
sessions using rdb-channel + streaming rdb directly from the background
process to the replica.
First graph- with at most 50 clients and light weight commands, we saw
5%-7.5% improvement in write latency during sync session.
Two graphs below- full sync was tested during heavy read commands from
the primary (such as sdiff, sunion on large sets). In that case, the
child process writes to the replica without sharing CPU with the loaded
main process. As a result, this not only improves client response time,
but may also shorten sync time by about 50%. The shorter sync time
results in less memory being used to store replication diffs (>60% in
some of the tested cases).
## Test setup
Both primary and replica in the performance tests ran on the same
machine. RDB size in all tests is 3.7gb. I generated write load using
valkey-benchmark ` ./valkey-benchmark -r 100000 -n 6000000 lpush my_list
__rand_int__`.
---------
Signed-off-by: naglera <anagler123@gmail.com>
Signed-off-by: naglera <58042354+naglera@users.noreply.github.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Ping Xie <pingxie@outlook.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-07-17 23:59:33 +03:00
|
|
|
assert {[status $master sync_partial_ok] == $psync_count}
|
2021-09-13 15:39:11 +08:00
|
|
|
assert {[status $master sync_full] == 1}
|
Replication backlog and replicas use one global shared replication buffer (#9166)
## Background
For redis master, one replica uses one copy of replication buffer, that is a big waste of memory,
more replicas more waste, and allocate/free memory for every reply list also cost much.
If we set client-output-buffer-limit small and write traffic is heavy, master may disconnect with
replicas and can't finish synchronization with replica. If we set client-output-buffer-limit big,
master may be OOM when there are many replicas that separately keep much memory.
Because replication buffers of different replica client are the same, one simple idea is that
all replicas only use one replication buffer, that will effectively save memory.
Since replication backlog content is the same as replicas' output buffer, now we
can discard replication backlog memory and use global shared replication buffer
to implement replication backlog mechanism.
## Implementation
I create one global "replication buffer" which contains content of replication stream.
The structure of "replication buffer" is similar to the reply list that exists in every client.
But the node of list is `replBufBlock`, which has `id, repl_offset, refcount` fields.
```c
/* Replication buffer blocks is the list of replBufBlock.
*
* +--------------+ +--------------+ +--------------+
* | refcount = 1 | ... | refcount = 0 | ... | refcount = 2 |
* +--------------+ +--------------+ +--------------+
* | / \
* | / \
* | / \
* Repl Backlog Replia_A Replia_B
*
* Each replica or replication backlog increments only the refcount of the
* 'ref_repl_buf_node' which it points to. So when replica walks to the next
* node, it should first increase the next node's refcount, and when we trim
* the replication buffer nodes, we remove node always from the head node which
* refcount is 0. If the refcount of the head node is not 0, we must stop
* trimming and never iterate the next node. */
/* Similar with 'clientReplyBlock', it is used for shared buffers between
* all replica clients and replication backlog. */
typedef struct replBufBlock {
int refcount; /* Number of replicas or repl backlog using. */
long long id; /* The unique incremental number. */
long long repl_offset; /* Start replication offset of the block. */
size_t size, used;
char buf[];
} replBufBlock;
```
So now when we feed replication stream into replication backlog and all replicas, we only need
to feed stream into replication buffer `feedReplicationBuffer`. In this function, we set some fields of
replication backlog and replicas to references of the global replication buffer blocks. And we also
need to check replicas' output buffer limit to free if exceeding `client-output-buffer-limit`, and trim
replication backlog if exceeding `repl-backlog-size`.
When sending reply to replicas, we also need to iterate replication buffer blocks and send its
content, when totally sending one block for replica, we decrease current node count and
increase the next current node count, and then free the block which reference is 0 from the
head of replication buffer blocks.
Since now we use linked list to manage replication backlog, it may cost much time for iterating
all linked list nodes to find corresponding replication buffer node. So we create a rax tree to
store some nodes for index, but to avoid rax tree occupying too much memory, i record
one per 64 nodes for index.
Currently, to make partial resynchronization as possible as much, we always let replication
backlog as the last reference of replication buffer blocks, backlog size may exceeds our setting
if slow replicas that reference vast replication buffer blocks, and this method doesn't increase
memory usage since they share replication buffer. To avoid freezing server for freeing unreferenced
replication buffer blocks when we need to trim backlog for exceeding backlog size setting,
we trim backlog incrementally (free 64 blocks per call now), and make it faster in
`beforeSleep` (free 640 blocks).
### Other changes
- `mem_total_replication_buffers`: we add this field in INFO command, it means the total
memory of replication buffers used.
- `mem_clients_slaves`: now even replica is slow to replicate, and its output buffer memory
is not 0, but it still may be 0, since replication backlog and replicas share one global replication
buffer, only if replication buffer memory is more than the repl backlog setting size, we consider
the excess as replicas' memory. Otherwise, we think replication buffer memory is the consumption
of repl backlog.
- Key eviction
Since all replicas and replication backlog share global replication buffer, we think only the
part of exceeding backlog size the extra separate consumption of replicas.
Because we trim backlog incrementally in the background, backlog size may exceeds our
setting if slow replicas that reference vast replication buffer blocks disconnect.
To avoid massive eviction loop, we don't count the delayed freed replication backlog into
used memory even if there are no replicas, i.e. we also regard this memory as replicas's memory.
- `client-output-buffer-limit` check for replica clients
It doesn't make sense to set the replica clients output buffer limit lower than the repl-backlog-size
config (partial sync will succeed and then replica will get disconnected). Such a configuration is
ignored (the size of repl-backlog-size will be used). This doesn't have memory consumption
implications since the replica client will share the backlog buffers memory.
- Drop replication backlog after loading data if needed
We always create replication backlog if server is a master, we need it because we put DELs in
it when loading expired keys in RDB, but if RDB doesn't have replication info or there is no rdb,
it is not possible to support partial resynchronization, to avoid extra memory of replication backlog,
we drop it.
- Multi IO threads
Since all replicas and replication backlog use global replication buffer, if I/O threads are enabled,
to guarantee data accessing thread safe, we must let main thread handle sending the output buffer
to all replicas. But before, other IO threads could handle sending output buffer of all replicas.
## Other optimizations
This solution resolve some other problem:
- When replicas disconnect with master since of out of output buffer limit, releasing the output
buffer of replicas may freeze server if we set big `client-output-buffer-limit` for replicas, but now,
it doesn't cause freezing.
- This implementation may mitigate reply list copy cost time(also freezes server) when one replication
has huge reply buffer and another replica can copy buffer for full synchronization. now, we just copy
reference info, it is very light.
- If we set replication backlog size big, it also may cost much time to copy replication backlog into
replica's output buffer. But this commit eliminates this problem.
- Resizing replication backlog size doesn't empty current replication backlog content.
2021-10-25 14:24:31 +08:00
|
|
|
assert {[status $master rdb_last_load_keys_expired] == 2048}
|
2021-09-13 15:39:11 +08:00
|
|
|
assert {[status $replica sync_full] == 1}
|
|
|
|
|
|
|
|
|
|
set digest [$master debug digest]
|
|
|
|
|
assert {$digest eq [$replica debug digest]}
|
|
|
|
|
assert {$digest eq [$sub_replica debug digest]}
|
|
|
|
|
}
|
|
|
|
|
}}}
|