From 598c324bc16839776d89af6550f594cb00e2cce1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 18:53:12 +0100 Subject: [PATCH 1/2] Setting N I/O threads should mean N-1 additional + 1 main thread. --- src/networking.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/networking.c b/src/networking.c index 73dc4afca..a2e454d4b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2658,9 +2658,9 @@ int io_threads_active; /* Are the threads currently spinning waiting I/O? */ int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */ /* This is the list of clients each thread will serve when threaded I/O is - * used. We spawn N threads, and the N+1 list is used for the clients that - * are processed by the main thread itself (this is why ther is "+1"). */ -list *io_threads_list[IO_THREADS_MAX_NUM+1]; + * used. We spawn io_threads_num-1 threads, since one is the main thread + * itself. */ +list *io_threads_list[IO_THREADS_MAX_NUM]; void *IOThreadMain(void *myid) { /* The ID is the thread number (from 0 to server.iothreads_num-1), and is @@ -2720,12 +2720,16 @@ void initThreadedIO(void) { exit(1); } - /* Spawn the I/O threads. */ + /* Spawn and initialize the I/O threads. */ for (int i = 0; i < server.io_threads_num; i++) { + /* Things we do for all the threads including the main thread. */ + io_threads_list[i] = listCreate(); + if (i == 0) continue; /* Thread 0 is the main thread. */ + + /* Things we do only for the additional threads. */ pthread_t tid; pthread_mutex_init(&io_threads_mutex[i],NULL); io_threads_pending[i] = 0; - io_threads_list[i] = listCreate(); pthread_mutex_lock(&io_threads_mutex[i]); /* Thread will be stopped. */ if (pthread_create(&tid,NULL,IOThreadMain,(void*)(long)i) != 0) { serverLog(LL_WARNING,"Fatal: Can't initialize IO thread."); @@ -2733,14 +2737,13 @@ void initThreadedIO(void) { } io_threads[i] = tid; } - io_threads_list[server.io_threads_num] = listCreate(); /* For main thread */ } void startThreadedIO(void) { if (tio_debug) { printf("S"); fflush(stdout); } if (tio_debug) printf("--- STARTING THREADED IO ---\n"); serverAssert(io_threads_active == 0); - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pthread_mutex_unlock(&io_threads_mutex[j]); io_threads_active = 1; } @@ -2754,7 +2757,7 @@ void stopThreadedIO(void) { (int) listLength(server.clients_pending_read), (int) listLength(server.clients_pending_write)); serverAssert(io_threads_active == 1); - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pthread_mutex_lock(&io_threads_mutex[j]); io_threads_active = 0; } @@ -2805,7 +2808,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { while((ln = listNext(&li))) { client *c = listNodeValue(ln); c->flags &= ~CLIENT_PENDING_WRITE; - int target_id = item_id % (server.io_threads_num+1); + int target_id = item_id % server.io_threads_num; listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2813,23 +2816,23 @@ int handleClientsWithPendingWritesUsingThreads(void) { /* Give the start condition to the waiting threads, by setting the * start condition atomic var. */ io_threads_op = IO_THREADS_OP_WRITE; - for (int j = 0; j < server.io_threads_num; j++) { + for (int j = 1; j < server.io_threads_num; j++) { int count = listLength(io_threads_list[j]); io_threads_pending[j] = count; } - /* Also use the main thread to process a slide of clients. */ - listRewind(io_threads_list[server.io_threads_num],&li); + /* Also use the main thread to process a slice of clients. */ + listRewind(io_threads_list[0],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); writeToClient(c,0); } - listEmpty(io_threads_list[server.io_threads_num]); + listEmpty(io_threads_list[0]); - /* Wait for all threads to end their work. */ + /* Wait for all the other threads to end their work. */ while(1) { unsigned long pending = 0; - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pending += io_threads_pending[j]; if (pending == 0) break; } @@ -2890,7 +2893,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { int item_id = 0; while((ln = listNext(&li))) { client *c = listNodeValue(ln); - int target_id = item_id % (server.io_threads_num+1); + int target_id = item_id % server.io_threads_num; listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2898,23 +2901,23 @@ int handleClientsWithPendingReadsUsingThreads(void) { /* Give the start condition to the waiting threads, by setting the * start condition atomic var. */ io_threads_op = IO_THREADS_OP_READ; - for (int j = 0; j < server.io_threads_num; j++) { + for (int j = 1; j < server.io_threads_num; j++) { int count = listLength(io_threads_list[j]); io_threads_pending[j] = count; } - /* Also use the main thread to process a slide of clients. */ - listRewind(io_threads_list[server.io_threads_num],&li); + /* Also use the main thread to process a slice of clients. */ + listRewind(io_threads_list[0],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); readQueryFromClient(c->conn); } - listEmpty(io_threads_list[server.io_threads_num]); + listEmpty(io_threads_list[0]); - /* Wait for all threads to end their work. */ + /* Wait for all the other threads to end their work. */ while(1) { unsigned long pending = 0; - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pending += io_threads_pending[j]; if (pending == 0) break; } From 9a39a227fff45baa0f1532dba017f31ac71b68e5 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 19:10:42 +0100 Subject: [PATCH 2/2] Document I/O threads in redis.conf. --- redis.conf | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/redis.conf b/redis.conf index 5f9547c1d..07005cffe 100644 --- a/redis.conf +++ b/redis.conf @@ -938,6 +938,52 @@ lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speedup the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usually. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis theads, otherwise you'll not +# be able to notice the improvements. + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is