diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..847abcf02 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +on: [push, pull_request] + +jobs: + build-ubuntu: + strategy: + matrix: + platform: [ubuntu-latest, ubuntu-16.04] + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v1 + - name: make + run: make + - name: test + run: | + sudo apt-get install tcl8.5 + make test + + build-macos-latest: + strategy: + matrix: + platform: [macos-latest, macOS-10.14] + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v1 + - name: make + run: make diff --git a/.gitignore b/.gitignore index f6e6abb12..c54671d59 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .*.swp core *.o +*.xo +*.so *.d *.log dump.rdb @@ -37,3 +39,4 @@ deps/lua/src/liblua.a .prerequisites *.dSYM Makefile.dep +.vscode/* diff --git a/00-RELEASENOTES b/00-RELEASENOTES index ce472159e..3572bcc54 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,16 +1,130 @@ -Hello! This file is just a placeholder, since this is the "unstable" branch -of Redis, the place where all the development happens. +Redis 6.0 release notes +======================= -There is no release notes for this branch, it gets forked into another branch -every time there is a partial feature freeze in order to eventually create -a new stable release. +-------------------------------------------------------------------------------- +Upgrade urgency levels: -Usually "unstable" is stable enough for you to use it in development environments -however you should never use it in production environments. It is possible -to download the latest stable release here: +LOW: No need to upgrade unless there are new features you want to use. +MODERATE: Program an upgrade of the server, but it's not urgent. +HIGH: There is a critical bug that may affect a subset of users. Upgrade! +CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. +SECURITY: There are security fixes in the release. +-------------------------------------------------------------------------------- - http://download.redis.io/releases/redis-stable.tar.gz +================================================================================ +Redis 6.0 RC1 Released Thu Dec 19 09:58:24 CEST 2019 +================================================================================ -More information is available at http://redis.io +Upgrade urgency LOW: This is the first RC of Redis 6. -Happy hacking! +Introduction to the Redis 6 release +=================================== + +Redis 6 improves Redis in a number of key areas and is one of the largest +Redis releases in the history of the project, so here we'll list only +the biggest features in this release: + +* The modules system now has a number of new APIs that allow module authors + to make things otherwise not possible in the past. It is possible to + store arbitrary module private data in RDB files, to hook on different + server events, capture and rewrite commands executions, block clients on + keys, and so forth. +* The Redis active expire cycle was rewritten for much faster eviction of keys + that are already expired. Now the effort is tunable. +* Redis now supports SSL on all channels. +* ACL support, you can define users that can run only certain commands and/or + can only access only certain keys patterns. +* Redis now supports a new protocol called RESP3, which returns more + semantical replies: new clients using this protocol can understand just + from the reply what type to return to the calling program. +* There is server-side support for client-side caching of key values. This + feature is still experimental and will get more changes during the next + release candidates, but you can already test it and read about it here: + https://redis.io/topics/client-side-caching +* Redis can now optionally use threads to handle I/O, allowing to serve + 2 times as much operations per second in a single instance when + pipelining cannot be used. +* Diskless replication is now supported even on replicas: a replica is now + able, under certain conditions the user can configure, to load the RDB + in the first synchronization directly from the socket to the memory. +* Redis-benchmark now supports a Redis Cluster mode. +* SRANDMEMBER and similar commands have a better distribution. +* Redis-cli improvements. +* Systemd support rewritten. +* A Redis Cluster proxy was released here: + https://github.com/artix75/redis-cluster-proxy +* A Disque module for Redis was released here: + https://github.com/antirez/disque-module + +Thanks to all the users and developers who made this release possible. +We'll follow up with more RC releases, until the code looks production ready +and we don't get reports of serious issues for a while. + +A special thank you for the amount of work put into this release +(in decreasing number of commits, only listing contributors with more +than a single commit) by: + + 685 antirez + 81 zhaozhao.zz + 76 Oran Agra + 51 artix + 28 Madelyn Olson + 27 Yossi Gottlieb + 15 David Carlier + 14 Guy Benoish + 14 Guy Korland + 13 Itamar Haber + 9 Angus Pearson + 8 WuYunlong + 8 yongman + 7 vattezhang + 7 Chris Lamb + 5 Dvir Volk + 5 meir@redislabs.com + 5 chendianqiang + 5 John Sully + 4 dejun.xdj + 4 Daniel Dai + 4 Johannes Truschnigg + 4 swilly22 + 3 Bruce Merry + 3 filipecosta90 + 3 youjiali1995 + 2 James Rouzier + 2 Andrey Bugaevskiy + 2 Brad Solomon + 2 Hamid Alaei + 2 Michael Chaten + 2 Steve Webster + 2 Wander Hillen + 2 Weiliang Li + 2 Yuan Zhou + 2 charsyam + 2 hujie + 2 jem + 2 shenlongxing + 2 valentino + 2 zhudacai 00228490 + 2 喜欢兰花山丘 + +Migrating from 5.0 to 6.0 +========================= + +Redis 6.0 is mostly a strict superset of 5.0, you should not have any problem +upgrading your application from 5.0 to 6.0. However this is a list of small +non-backward compatible changes introduced in the 6.0 release: + +* Nothing found yet. + +-------------------------------------------------------------------------------- + +Credits: For each release, a list of changes with the relative author is +provided. Where not specified the implementation and design is done by +Salvatore Sanfilippo. Thanks to Redis Labs for making all this possible. +Also many thanks to all the other contributors and the amazing community +we have. + +Commit messages may contain additional credits. + +Enjoy, +Salvatore diff --git a/README.md b/README.md index 70b879d52..c06a32d6c 100644 --- a/README.md +++ b/README.md @@ -251,8 +251,49 @@ $ docker run -it --rm -v /path-to-dump-binaries:/keydb_bin eqalpha/keydb-build-b ``` Please note that you will need libcurl4-openssl-dev in order to run keydb. With flash version you may need libnuma-dev and libtool installed in order to run the binaries. Keep this in mind especially when running in a container. For a copy of all our Dockerfiles, please see them on [docs]( https://docs.keydb.dev/docs/dockerfiles/). +<<<<<<< HEAD Code contributions ----------------- +======= +One of the most important functions inside this file is `replicationFeedSlaves()` that writes commands to the clients representing replica instances connected +to our master, so that the replicas can get the writes performed by the clients: +this way their data set will remain synchronized with the one in the master. + +This file also implements both the `SYNC` and `PSYNC` commands that are +used in order to perform the first synchronization between masters and +replicas, or to continue the replication after a disconnection. + +Other C files +--- + +* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c`, `t_zset.c` and `t_stream.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types. +* `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand. +* `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information. +* `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel. +* `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally. +* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familar with the Lua API. +* `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3]. + +[3]: http://redis.io/topics/cluster-spec + +Anatomy of a Redis command +--- + +All the Redis commands are defined in the following way: + + void foobarCommand(client *c) { + printf("%s",c->argv[1]->ptr); /* Do something with the argument. */ + addReply(c,shared.ok); /* Reply something to the client. */ + } + +The command is then referenced inside `server.c` in the command table: + + {"foobar",foobarCommand,2,"rtF",0,NULL,0,0,0,0,0}, + +In the above example `2` is the number of arguments the command takes, +while `"rtF"` are the command flags, as documented in the command table +top comment inside `server.c`. +>>>>>>> redis/6.0 Note: by contributing code to the KeyDB project in any form, including sending a pull request via Github, a code fragment or patch via private email or diff --git a/TLS.md b/TLS.md new file mode 100644 index 000000000..76fe0be2e --- /dev/null +++ b/TLS.md @@ -0,0 +1,106 @@ +TLS Support -- Work In Progress +=============================== + +This is a brief note to capture current thoughts/ideas and track pending action +items. + +Getting Started +--------------- + +### Building + +To build with TLS support you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu). + +Run `make BUILD_TLS=yes`. + +### Tests + +To run Redis test suite with TLS, you'll need TLS support for TCL (i.e. +`tcl-tls` package on Debian/Ubuntu). + +1. Run `./utils/gen-test-certs.sh` to generate a root CA and a server + certificate. + +2. Run `./runtest --tls` or `./runtest-cluster --tls` to run Redis and Redis + Cluster tests in TLS mode. + +### Running manually + +To manually run a Redis server with TLS mode (assuming `gen-test-certs.sh` was +invoked so sample certificates/keys are available): + + ./src/redis-server --tls-port 6379 --port 0 \ + --tls-cert-file ./tests/tls/redis.crt \ + --tls-key-file ./tests/tls/redis.key \ + --tls-ca-cert-file ./tests/tls/ca.crt + +To connect to this Redis server with `redis-cli`: + + ./src/redis-cli --tls \ + --cert ./tests/tls/redis.crt \ + --key ./tests/tls/redis.key \ + --cacert ./tests/tls/ca.crt + +This will disable TCP and enable TLS on port 6379. It's also possible to have +both TCP and TLS available, but you'll need to assign different ports. + +To make a Replica connect to the master using TLS, use `--tls-replication yes`, +and to make Redis Cluster use TLS across nodes use `--tls-cluster yes`. + +Connections +----------- + +All socket operations now go through a connection abstraction layer that hides +I/O and read/write event handling from the caller. + +**Multi-threading I/O is not currently supported for TLS**, as a TLS connection +needs to do its own manipulation of AE events which is not thread safe. The +solution is probably to manage independent AE loops for I/O threads and longer +term association of connections with threads. This may potentially improve +overall performance as well. + +Sync IO for TLS is currently implemented in a hackish way, i.e. making the +socket blocking and configuring socket-level timeout. This means the timeout +value may not be so accurate, and there would be a lot of syscall overhead. +However I believe that getting rid of syncio completely in favor of pure async +work is probably a better move than trying to fix that. For replication it would +probably not be so hard. For cluster keys migration it might be more difficult, +but there are probably other good reasons to improve that part anyway. + +To-Do List +========== + +Additional TLS Features +----------------------- + +1. Add metrics to INFO? +2. Add session caching support. Check if/how it's handled by clients to assess + how useful/important it is. + +redis-benchmark +--------------- + +The current implementation is a mix of using hiredis for parsing and basic +networking (establishing connections), but directly manipulating sockets for +most actions. + +This will need to be cleaned up for proper TLS support. The best approach is +probably to migrate to hiredis async mode. + +redis-cli +--------- + +1. Add support for TLS in --slave and --rdb modes. + +Others +------ + +Consider the implications of allowing TLS to be configured on a separate port, +making Redis listening on multiple ports. + +This impacts many things, like +1. Startup banner port notification +2. Proctitle +3. How slaves announce themselves +4. Cluster bus port calculation diff --git a/deps/Makefile b/deps/Makefile index bacc8f48e..d76693622 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -41,9 +41,13 @@ distclean: .PHONY: distclean +ifeq ($(BUILD_TLS),yes) + HIREDIS_MAKE_FLAGS = USE_SSL=1 +endif + hiredis: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd hiredis && $(MAKE) static + cd hiredis && $(MAKE) static $(HIREDIS_MAKE_FLAGS) .PHONY: hiredis diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index c44b5c537..8e50b5434 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -5,3 +5,4 @@ /*.dylib /*.a /*.pc +*.dSYM diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index faf2ce684..dd8e0e73d 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -26,20 +26,72 @@ addons: - libc6-dev-i386 - libc6-dbg:i386 - gcc-multilib + - g++-multilib - valgrind env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + - BITS="32" + - BITS="64" + +script: + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + else + TEST_PREFIX="valgrind --track-origins=yes --leak-check=full"; + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + fi; + export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - mkdir build/ && cd build/ + - cmake .. ${EXTRA_CMAKE_OPTS} + - make VERBOSE=1 + - ctest -V matrix: - exclude: - - os: osx - env: PRE="valgrind --track-origins=yes --leak-check=full" + include: + # Windows MinGW cross compile on Linux + - os: linux + dist: xenial + compiler: mingw + addons: + apt: + packages: + - ninja-build + - gcc-mingw-w64-x86-64 + - g++-mingw-w64-x86-64 + script: + - mkdir build && cd build + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on + - ninja -v - - os: osx - env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" - -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example + # Windows MSVC 2017 + - os: windows + compiler: msvc + env: + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + before_install: + - eval "${MATRIX_EVAL}" + install: + - choco install ninja + script: + - mkdir build && cd build + - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && + ninja -v' + - ctest -V diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index a7fe3ac11..d1d37e515 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -12,6 +12,16 @@ compare to other values, casting might be necessary or can be removed, if casting was applied before. +### 0.x.x (unreleased) +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. +If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + +* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. + ### 0.14.0 (2018-09-25) * Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) @@ -50,8 +60,9 @@ * Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 * Fix warnings, when compiled with -Wshadow * Make hiredis compile in Cygwin on Windows, now CI-tested - -**BREAKING CHANGES**: +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. * Remove backwards compatibility macro's diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt new file mode 100644 index 000000000..9e78894f3 --- /dev/null +++ b/deps/hiredis/CMakeLists.txt @@ -0,0 +1,90 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) +INCLUDE(GNUInstallDirs) +PROJECT(hiredis) + +OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) + +MACRO(getVersionBit name) + SET(VERSION_REGEX "^#define ${name} (.+)$") + FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h" + VERSION_BIT REGEX ${VERSION_REGEX}) + STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}") +ENDMACRO(getVersionBit) + +getVersionBit(HIREDIS_MAJOR) +getVersionBit(HIREDIS_MINOR) +getVersionBit(HIREDIS_PATCH) +getVersionBit(HIREDIS_SONAME) +SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") +MESSAGE("Detected version: ${VERSION}") + +PROJECT(hiredis VERSION "${VERSION}") + +SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") + +ADD_LIBRARY(hiredis SHARED + async.c + dict.c + hiredis.c + net.c + read.c + sds.c + sockcompat.c) + +SET_TARGET_PROPERTIES(hiredis + PROPERTIES + VERSION "${HIREDIS_SONAME}") +IF(WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) +ENDIF() +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) + +INSTALL(TARGETS hiredis + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +INSTALL(FILES hiredis.h read.h sds.h async.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(DIRECTORY adapters + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +IF(ENABLE_SSL) + IF (NOT OPENSSL_ROOT_DIR) + IF (APPLE) + SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + ENDIF() + ENDIF() + FIND_PACKAGE(OpenSSL REQUIRED) + ADD_LIBRARY(hiredis_ssl SHARED + ssl.c) + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) + + INSTALL(TARGETS hiredis_ssl + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + INSTALL(FILES hiredis_ssl.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +ENDIF() + +IF(NOT (WIN32 OR MINGW)) + ENABLE_TESTING() + ADD_EXECUTABLE(hiredis-test test.c) + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ADD_TEST(NAME hiredis-test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) +ENDIF() + +# Add examples +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF(ENABLE_EXAMPLES) diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index aac8aa06c..841990d5c 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,11 +3,17 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o +OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +SSL_OBJ=ssl.o EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +ifeq ($(USE_SSL),1) +EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl +endif TESTS=hiredis-test LIBNAME=libhiredis +SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -39,7 +45,7 @@ export REDIS_TEST_CONFIG CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc') CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers DEBUG_FLAGS?= -g -ggdb REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) REAL_LDFLAGS=$(LDFLAGS) @@ -49,12 +55,30 @@ STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +STLIB_MAKE_CMD=$(AR) rcs # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +USE_SSL?=0 + +# This is required for test.c only +ifeq ($(USE_SSL),1) + CFLAGS+=-DHIREDIS_TEST_SSL +endif + +ifeq ($(uname_S),Linux) + SSL_LDFLAGS=-lssl -lcrypto +else + OPENSSL_PREFIX?=/usr/local/opt/openssl + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +endif + ifeq ($(uname_S),SunOS) REAL_LDFLAGS+= -ldl -lnsl -lsocket DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) @@ -66,40 +90,61 @@ ifeq ($(uname_S),Darwin) endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) +ifeq ($(USE_SSL),1) +all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) +endif # Deps (use make dep to generate this) async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h read.o: read.c fmacros.h read.h sds.h sds.o: sds.c sds.h +sockcompat.o: sockcompat.c sockcompat.h +ssl.o: ssl.c hiredis.h test.o: test.c fmacros.h hiredis.h read.h sds.h $(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) + $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) $(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) + $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) + +$(SSL_DYLIBNAME): $(SSL_OBJ) + $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + +$(SSL_STLIBNAME): $(SSL_OBJ) + $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) dynamic: $(DYLIBNAME) static: $(STLIBNAME) +ifeq ($(USE_SSL),1) +dynamic: $(SSL_DYLIBNAME) +static: $(SSL_STLIBNAME) +endif # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) ifndef AE_DIR hiredis-example-ae: @@ -116,7 +161,7 @@ hiredis-example-libuv: @false else hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) @@ -133,32 +178,33 @@ hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) endif hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) examples: $(EXAMPLES) -hiredis-test: test.o $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) +ifeq ($(USE_SSL),1) + TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread +endif +hiredis-test: test.o $(TEST_LIBS) hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) test: hiredis-test ./hiredis-test check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` + TEST_SSL=$(USE_SSL) ./test.sh .c.o: $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c INSTALL?= cp -pPR @@ -175,6 +221,20 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ +$(SSL_PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis_ssl >> $@ + @echo Description: SSL Support for hiredis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Requires: hiredis >> $@ + @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ + @echo Libs.private: -lssl -lcrypto >> $@ + install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index 01223ea59..c0b432f07 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -286,6 +286,7 @@ return `REDIS_ERR`. The function to set the disconnect callback has the followin ```c int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); ``` +`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback. ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. @@ -406,6 +407,6 @@ as soon as possible in order to prevent allocation of useless memory. ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 7d2bef180..a4952776c 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -34,48 +34,113 @@ #include "../hiredis.h" #include "../async.h" +#define REDIS_LIBEVENT_DELETED 0x01 +#define REDIS_LIBEVENT_ENTERED 0x02 + typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event *rev, *wev; + struct event *ev; + struct event_base *base; + struct timeval tv; + short flags; + short state; } redisLibeventEvents; -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); +static void redisLibeventDestroy(redisLibeventEvents *e) { + free(e); } -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); +static void redisLibeventHandler(int fd, short event, void *arg) { + ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); + e->state |= REDIS_LIBEVENT_ENTERED; + + #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\ + redisLibeventDestroy(e);\ + return; \ + } + + if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleTimeout(e->context); + CHECK_DELETED(); + } + + if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleRead(e->context); + CHECK_DELETED(); + } + + if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleWrite(e->context); + CHECK_DELETED(); + } + + e->state &= ~REDIS_LIBEVENT_ENTERED; + #undef CHECK_DELETED +} + +static void redisLibeventUpdate(void *privdata, short flag, int isRemove) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL; + + if (isRemove) { + if ((e->flags & flag) == 0) { + return; + } else { + e->flags &= ~flag; + } + } else { + if (e->flags & flag) { + return; + } else { + e->flags |= flag; + } + } + + event_del(e->ev); + event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST, + redisLibeventHandler, privdata); + event_add(e->ev, tv); } static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->rev,NULL); + redisLibeventUpdate(privdata, EV_READ, 0); } static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); + redisLibeventUpdate(privdata, EV_READ, 1); } static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->wev,NULL); + redisLibeventUpdate(privdata, EV_WRITE, 0); } static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->wev); + redisLibeventUpdate(privdata, EV_WRITE, 1); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_free(e->rev); - event_free(e->wev); - free(e); + if (!e) { + return; + } + event_del(e->ev); + event_free(e->ev); + e->ev = NULL; + + if (e->state & REDIS_LIBEVENT_ENTERED) { + e->state |= REDIS_LIBEVENT_DELETED; + } else { + redisLibeventDestroy(e); + } +} + +static void redisLibeventSetTimeout(void *privdata, struct timeval tv) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + short flags = e->flags; + e->flags = 0; + e->tv = tv; + redisLibeventUpdate(e, flags, 0); } static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { @@ -87,7 +152,7 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); + e = (redisLibeventEvents*)calloc(1, sizeof(*e)); e->context = ac; /* Register functions to start/stop listening for events */ @@ -96,13 +161,12 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.addWrite = redisLibeventAddWrite; ac->ev.delWrite = redisLibeventDelWrite; ac->ev.cleanup = redisLibeventCleanup; + ac->ev.scheduleTimer = redisLibeventSetTimeout; ac->ev.data = e; /* Initialize and install read/write events */ - e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); - e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); - event_add(e->rev, NULL); - event_add(e->wev, NULL); + e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e); + e->base = base; return REDIS_OK; } #endif diff --git a/deps/hiredis/appveyor.yml b/deps/hiredis/appveyor.yml index 819efbd58..5b43fdbeb 100644 --- a/deps/hiredis/appveyor.yml +++ b/deps/hiredis/appveyor.yml @@ -5,8 +5,9 @@ environment: CC: gcc - CYG_BASH: C:\cygwin\bin\bash CC: gcc - TARGET: 32bit - TARGET_VARS: 32bit-vars + CFLAGS: -m32 + CXXFLAGS: -m32 + LDFLAGS: -m32 clone_depth: 1 @@ -20,4 +21,4 @@ install: build_script: - 'echo building...' - - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 #include +#ifndef _MSC_VER #include +#endif #include #include #include @@ -40,22 +42,9 @@ #include "net.h" #include "dict.c" #include "sds.h" +#include "win32.h" -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); +#include "async_private.h" /* Forward declaration of function in hiredis.c */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); @@ -126,6 +115,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->ev.addWrite = NULL; ac->ev.delWrite = NULL; ac->ev.cleanup = NULL; + ac->ev.scheduleTimer = NULL; ac->onConnect = NULL; ac->onDisconnect = NULL; @@ -150,56 +140,52 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { ac->errstr = c->errstr; } -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { + redisOptions myOptions = *options; redisContext *c; redisAsyncContext *ac; - c = redisConnectNonBlock(ip,port); - if (c == NULL) + myOptions.options |= REDIS_OPT_NONBLOCK; + c = redisConnectWithOptions(&myOptions); + if (c == NULL) { return NULL; - + } ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } - __redisAsyncCopyError(ac); return ac; } +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisAsyncConnectWithOptions(&options); +} + redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_REUSEADDR; + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisAsyncConnectWithOptions(&options); } int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { @@ -328,7 +314,7 @@ void redisAsyncFree(redisAsyncContext *ac) { } /* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { +void __redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); /* Make sure error is accessible if there is any */ @@ -344,9 +330,15 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { c->flags |= REDIS_DISCONNECTING; } + /* cleanup event library on disconnect. + * this is safe to call multiple times */ + _EL_CLEANUP(ac); + /* For non-clean disconnects, __redisAsyncFree() will execute pending * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); + if (!(c->flags & REDIS_NO_AUTO_FREE)) { + __redisAsyncFree(ac); + } } /* Tries to do a clean disconnect from Redis, meaning it stops new commands @@ -358,6 +350,9 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { void redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); c->flags |= REDIS_DISCONNECTING; + + /** unset the auto-free flag here, because disconnect undoes this */ + c->flags &= ~REDIS_NO_AUTO_FREE; if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) __redisAsyncDisconnect(ac); } @@ -408,7 +403,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, assert(reply->element[2]->type == REDIS_REPLY_INTEGER); /* Unset subscribed flag only when no pipelined pending subscribe. */ - if (reply->element[2]->integer == 0 + if (reply->element[2]->integer == 0 && dictSize(ac->sub.channels) == 0 && dictSize(ac->sub.patterns) == 0) c->flags &= ~REDIS_SUBSCRIBED; @@ -524,6 +519,18 @@ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { } } +void redisAsyncRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + /* This function should be called when the socket is readable. * It processes all replies that can be read and executes their callbacks. */ @@ -539,28 +546,13 @@ void redisAsyncHandleRead(redisAsyncContext *ac) { return; } - if (redisBufferRead(c) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Always re-schedule reads */ - _EL_ADD_READ(ac); - redisProcessCallbacks(ac); - } + c->funcs->async_read(ac); } -void redisAsyncHandleWrite(redisAsyncContext *ac) { +void redisAsyncWrite(redisAsyncContext *ac) { redisContext *c = &(ac->c); int done = 0; - if (!(c->flags & REDIS_CONNECTED)) { - /* Abort connect was not successful. */ - if (__redisAsyncHandleConnect(ac) != REDIS_OK) - return; - /* Try again later when the context is still not connected. */ - if (!(c->flags & REDIS_CONNECTED)) - return; - } - if (redisBufferWrite(c,&done) == REDIS_ERR) { __redisAsyncDisconnect(ac); } else { @@ -575,6 +567,51 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { } } +void redisAsyncHandleWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (!(c->flags & REDIS_CONNECTED)) { + /* Abort connect was not successful. */ + if (__redisAsyncHandleConnect(ac) != REDIS_OK) + return; + /* Try again later when the context is still not connected. */ + if (!(c->flags & REDIS_CONNECTED)) + return; + } + + c->funcs->async_write(ac); +} + +void __redisSetError(redisContext *c, int type, const char *str); + +void redisAsyncHandleTimeout(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + + if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; + } + + if (!c->err) { + __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + } + + if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { + ac->onConnect(ac, REDIS_ERR); + } + + while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) { + __redisRunCallback(ac, &cb, NULL); + } + + /** + * TODO: Don't automatically sever the connection, + * rather, allow to ignore responses before the queue is clear + */ + __redisAsyncDisconnect(ac); +} + /* Sets a pointer to the first argument and its length starting at p. Returns * the number of bytes to skip to get to the following argument. */ static const char *nextArgument(const char *start, const char **str, size_t *len) { @@ -714,3 +751,16 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.timeout) { + ac->c.timeout = calloc(1, sizeof(tv)); + } + + if (tv.tv_sec == ac->c.timeout->tv_sec && + tv.tv_usec == ac->c.timeout->tv_usec) { + return; + } + + *ac->c.timeout = tv; +} diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 740555c24..4f6b3b783 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -57,6 +57,7 @@ typedef struct redisCallbackList { /* Connection callback prototypes */ typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); +typedef void(redisTimerCallback)(void *timer, void *privdata); /* Context for an async connection to Redis */ typedef struct redisAsyncContext { @@ -81,6 +82,7 @@ typedef struct redisAsyncContext { void (*addWrite)(void *privdata); void (*delWrite)(void *privdata); void (*cleanup)(void *privdata); + void (*scheduleTimer)(void *privdata, struct timeval tv); } ev; /* Called when either the connection is terminated due to an error or per @@ -106,6 +108,7 @@ typedef struct redisAsyncContext { } redisAsyncContext; /* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options); redisAsyncContext *redisAsyncConnect(const char *ip, int port); redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, @@ -113,12 +116,17 @@ redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); /* Handle read/write events */ void redisAsyncHandleRead(redisAsyncContext *ac); void redisAsyncHandleWrite(redisAsyncContext *ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac); +void redisAsyncRead(redisAsyncContext *ac); +void redisAsyncWrite(redisAsyncContext *ac); /* Command functions for an async context. Write the command to the * output buffer and register the provided callback. */ diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h new file mode 100644 index 000000000..d0133ae18 --- /dev/null +++ b/deps/hiredis/async_private.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_PRIVATE_H +#define __HIREDIS_ASYNC_PRIVATE_H + +#define _EL_ADD_READ(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + ctx->ev.cleanup = NULL; \ + } while(0); + +static inline void refreshTimeout(redisAsyncContext *ctx) { + if (ctx->c.timeout && ctx->ev.scheduleTimer && + (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { + ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); + // } else { + // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); + // if (ctx->c.timeout){ + // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, + // ctx->c.timeout->tv_usec); + // } + } +} + +void __redisAsyncDisconnect(redisAsyncContext *ac); +void redisProcessCallbacks(redisAsyncContext *ac); + +#endif /* __HIREDIS_ASYNC_PRIVATE_H */ diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt new file mode 100644 index 000000000..dd3a313ac --- /dev/null +++ b/deps/hiredis/examples/CMakeLists.txt @@ -0,0 +1,46 @@ +INCLUDE(FindPkgConfig) +# Check for GLib + +PKG_CHECK_MODULES(GLIB2 glib-2.0) +if (GLIB2_FOUND) + INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) + ADD_EXECUTABLE(example-glib example-glib.c) + TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES}) +ENDIF(GLIB2_FOUND) + +FIND_PATH(LIBEV ev.h + HINTS /usr/local /usr/opt/local + ENV LIBEV_INCLUDE_DIR) + +if (LIBEV) + # Just compile and link with libev + ADD_EXECUTABLE(example-libev example-libev.c) + TARGET_LINK_LIBRARIES(example-libev hiredis ev) +ENDIF() + +FIND_PATH(LIBEVENT event.h) +if (LIBEVENT) + ADD_EXECUTABLE(example-libevent example-libevent) + TARGET_LINK_LIBRARIES(example-libevent hiredis event) +ENDIF() + +FIND_PATH(LIBUV uv.h) +IF (LIBUV) + ADD_EXECUTABLE(example-libuv example-libuv.c) + TARGET_LINK_LIBRARIES(example-libuv hiredis uv) +ENDIF() + +IF (APPLE) + FIND_LIBRARY(CF CoreFoundation) + ADD_EXECUTABLE(example-macosx example-macosx.c) + TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF}) +ENDIF() + +IF (ENABLE_SSL) + ADD_EXECUTABLE(example-ssl example-ssl.c) + TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl) +ENDIF() + +ADD_EXECUTABLE(example example.c) +TARGET_LINK_LIBRARIES(example hiredis) diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c new file mode 100644 index 000000000..1021113b9 --- /dev/null +++ b/deps/hiredis/examples/example-libevent-ssl.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + struct event_base *base = event_base_new(); + if (argc < 5) { + fprintf(stderr, + "Usage: %s [ca]\n", argv[0]); + exit(1); + } + + const char *value = argv[1]; + size_t nvalue = strlen(value); + + const char *hostname = argv[2]; + int port = atoi(argv[3]); + + const char *cert = argv[4]; + const char *certKey = argv[5]; + const char *caCert = argc > 5 ? argv[6] : NULL; + + redisAsyncContext *c = redisAsyncConnect(hostname, port); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + printf("SSL Error!\n"); + exit(1); + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + return 0; +} diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c index d333c22b7..1fe71ae4e 100644 --- a/deps/hiredis/examples/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -9,7 +9,12 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; + if (reply == NULL) { + if (c->errstr) { + printf("errstr: %s\n", c->errstr); + } + return; + } printf("argv[%s]: %s\n", (char*)privdata, reply->str); /* Disconnect after receiving the reply to GET */ @@ -35,8 +40,14 @@ void disconnectCallback(const redisAsyncContext *c, int status) { int main (int argc, char **argv) { signal(SIGPIPE, SIG_IGN); struct event_base *base = event_base_new(); + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + struct timeval tv = {0}; + tv.tv_sec = 1; + options.timeout = &tv; - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + + redisAsyncContext *c = redisAsyncConnectWithOptions(&options); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c new file mode 100644 index 000000000..81f4648c6 --- /dev/null +++ b/deps/hiredis/examples/example-ssl.c @@ -0,0 +1,97 @@ +#include +#include +#include + +#include +#include + +int main(int argc, char **argv) { + unsigned int j; + redisContext *c; + redisReply *reply; + if (argc < 4) { + printf("Usage: %s [ca]\n", argv[0]); + exit(1); + } + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = atoi(argv[2]); + const char *cert = argv[3]; + const char *key = argv[4]; + const char *ca = argc > 4 ? argv[5] : NULL; + + struct timeval tv = { 1, 500000 }; // 1.5 seconds + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, hostname, port); + options.timeout = &tv; + c = redisConnectWithOptions(&options); + + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + printf("Couldn't initialize SSL!\n"); + printf("Error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%u",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + return 0; +} diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 4d494c55a..0e93fc8b3 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -5,14 +5,27 @@ #include int main(int argc, char **argv) { - unsigned int j; + unsigned int j, isunix = 0; redisContext *c; redisReply *reply; const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + + if (argc > 2) { + if (*argv[2] == 'u' || *argv[2] == 'U') { + isunix = 1; + /* in this case, host is the path to the unix socket */ + printf("Will connect to unix socket @%s\n", hostname); + } + } + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); + if (isunix) { + c = redisConnectUnixWithTimeout(hostname, timeout); + } else { + c = redisConnectWithTimeout(hostname, port, timeout); + } if (c == NULL || c->err) { if (c) { printf("Connection error: %s\n", c->errstr); diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 0947d1ed7..abd94c01d 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -34,7 +34,6 @@ #include "fmacros.h" #include #include -#include #include #include #include @@ -42,10 +41,20 @@ #include "hiredis.h" #include "net.h" #include "sds.h" +#include "async.h" +#include "win32.h" + +static redisContextFuncs redisContextDefaultFuncs = { + .free_privdata = NULL, + .async_read = redisAsyncRead, + .async_write = redisAsyncWrite, + .read = redisNetRead, + .write = redisNetWrite +}; static redisReply *createReplyObject(int type); static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); +static void *createArrayObject(const redisReadTask *task, size_t elements); static void *createIntegerObject(const redisReadTask *task, long long value); static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len); static void *createNilObject(const redisReadTask *task); @@ -112,21 +121,34 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len if (r == NULL) return NULL; - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } - assert(task->type == REDIS_REPLY_ERROR || task->type == REDIS_REPLY_STATUS || - task->type == REDIS_REPLY_STRING); + task->type == REDIS_REPLY_STRING || + task->type == REDIS_REPLY_VERB); /* Copy string value */ - memcpy(buf,str,len); - buf[len] = '\0'; + if (task->type == REDIS_REPLY_VERB) { + buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(r->vtype,str,3); + r->vtype[3] = '\0'; + memcpy(buf,str+4,len-4); + buf[len-4] = '\0'; + r->len = len-4; + } else { + buf = malloc(len+1); + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(buf,str,len); + buf[len] = '\0'; + r->len = len; + } r->str = buf; - r->len = len; if (task->parent) { parent = task->parent->obj; @@ -138,7 +160,7 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len return r; } -static void *createArrayObject(const redisReadTask *task, int elements) { +static void *createArrayObject(const redisReadTask *task, size_t elements) { redisReply *r, *parent; r = createReplyObject(task->type); @@ -649,29 +671,30 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(void) { +static redisContext *redisContextInit(const redisOptions *options) { redisContext *c; - c = calloc(1,sizeof(redisContext)); + c = calloc(1, sizeof(*c)); if (c == NULL) return NULL; + c->funcs = &redisContextDefaultFuncs; c->obuf = sdsempty(); c->reader = redisReaderCreate(); + c->fd = REDIS_INVALID_FD; if (c->obuf == NULL || c->reader == NULL) { redisFree(c); return NULL; } - + (void)options; /* options are used in other functions */ return c; } void redisFree(redisContext *c) { if (c == NULL) return; - if (c->fd > 0) - close(c->fd); + redisNetClose(c); sdsfree(c->obuf); redisReaderFree(c->reader); @@ -680,12 +703,16 @@ void redisFree(redisContext *c) { free(c->unix_sock.path); free(c->timeout); free(c->saddr); + if (c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + } + memset(c, 0xff, sizeof(*c)); free(c); } -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; +redisFD redisFreeKeepFd(redisContext *c) { + redisFD fd = c->fd; + c->fd = REDIS_INVALID_FD; redisFree(c); return fd; } @@ -694,10 +721,13 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->fd > 0) { - close(c->fd); + if (c->privdata && c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + c->privdata = NULL; } + redisNetClose(c); + sdsfree(c->obuf); redisReaderFree(c->reader); @@ -718,112 +748,107 @@ int redisReconnect(redisContext *c) { return REDIS_ERR; } +redisContext *redisConnectWithOptions(const redisOptions *options) { + redisContext *c = redisContextInit(options); + if (c == NULL) { + return NULL; + } + if (!(options->options & REDIS_OPT_NONBLOCK)) { + c->flags |= REDIS_BLOCK; + } + if (options->options & REDIS_OPT_REUSEADDR) { + c->flags |= REDIS_REUSEADDR; + } + if (options->options & REDIS_OPT_NOAUTOFREE) { + c->flags |= REDIS_NO_AUTO_FREE; + } + + if (options->type == REDIS_CONN_TCP) { + redisContextConnectBindTcp(c, options->endpoint.tcp.ip, + options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.source_addr); + } else if (options->type == REDIS_CONN_UNIX) { + redisContextConnectUnix(c, options->endpoint.unix_socket, + options->timeout); + } else if (options->type == REDIS_CONN_USERFD) { + c->fd = options->endpoint.fd; + c->flags |= REDIS_CONNECTED; + } else { + // Unknown type - FIXME - FREE + return NULL; + } + if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->timeout); + } + return c; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisConnectWithOptions(&options); } redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; +redisContext *redisConnectFd(redisFD fd) { + redisOptions options = {0}; + options.type = REDIS_CONN_USERFD; + options.endpoint.fd = fd; + return redisConnectWithOptions(&options); } /* Set read/write timeout on a blocking socket. */ @@ -853,22 +878,15 @@ int redisBufferRead(redisContext *c) { if (c->err) return REDIS_ERR; - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ + nread = c->funcs->read(c, buf, sizeof(buf)); + if (nread > 0) { + if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); + return REDIS_ERR; } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + } else if (nread < 0) { return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } } return REDIS_OK; } @@ -883,21 +901,15 @@ int redisBufferRead(redisContext *c) { * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { - int nwritten; /* Return early when the context has seen an error. */ if (c->err) return REDIS_ERR; if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } + int nwritten = c->funcs->write(c); + if (nwritten < 0) { + return REDIS_ERR; } else if (nwritten > 0) { if (nwritten == (signed)sdslen(c->obuf)) { sdsfree(c->obuf); diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index 47d7982e9..69dc39c5e 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -35,7 +35,11 @@ #define __HIREDIS_H #include "read.h" #include /* for va_list */ +#ifndef _MSC_VER #include /* for struct timeval */ +#else +struct timeval; /* forward declaration */ +#endif #include /* uintXX_t, etc */ #include "sds.h" /* for sds */ @@ -74,6 +78,12 @@ /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/** + * Flag that indicates the user does not want the context to + * be automatically freed upon error + */ +#define REDIS_NO_AUTO_FREE 0x200 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and @@ -92,6 +102,8 @@ typedef struct redisReply { size_t len; /* Length of string */ char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING and REDIS_REPLY_DOUBLE (in additionl to dval). */ + char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null + terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ } redisReply; @@ -111,14 +123,93 @@ void redisFreeSdsCommand(sds cmd); enum redisConnectionType { REDIS_CONN_TCP, - REDIS_CONN_UNIX + REDIS_CONN_UNIX, + REDIS_CONN_USERFD }; +struct redisSsl; + +#define REDIS_OPT_NONBLOCK 0x01 +#define REDIS_OPT_REUSEADDR 0x02 + +/** + * Don't automatically free the async object on a connection failure, + * or other implicit conditions. Only free on an explicit call to disconnect() or free() + */ +#define REDIS_OPT_NOAUTOFREE 0x04 + +/* In Unix systems a file descriptor is a regular signed int, with -1 + * representing an invalid descriptor. In Windows it is a SOCKET + * (32- or 64-bit unsigned integer depending on the architecture), where + * all bits set (~0) is INVALID_SOCKET. */ +#ifndef _WIN32 +typedef int redisFD; +#define REDIS_INVALID_FD -1 +#else +#ifdef _WIN64 +typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ +#else +typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ +#endif +#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ +#endif + +typedef struct { + /* + * the type of connection to use. This also indicates which + * `endpoint` member field to use + */ + int type; + /* bit field of REDIS_OPT_xxx */ + int options; + /* timeout value. if NULL, no timeout is used */ + const struct timeval *timeout; + union { + /** use this field for tcp/ip connections */ + struct { + const char *source_addr; + const char *ip; + int port; + } tcp; + /** use this field for unix domain sockets */ + const char *unix_socket; + /** + * use this field to have hiredis operate an already-open + * file descriptor */ + redisFD fd; + } endpoint; +} redisOptions; + +/** + * Helper macros to initialize options to their specified fields. + */ +#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ + (opts)->type = REDIS_CONN_TCP; \ + (opts)->endpoint.tcp.ip = ip_; \ + (opts)->endpoint.tcp.port = port_; + +#define REDIS_OPTIONS_SET_UNIX(opts, path) \ + (opts)->type = REDIS_CONN_UNIX; \ + (opts)->endpoint.unix_socket = path; + +struct redisAsyncContext; +struct redisContext; + +typedef struct redisContextFuncs { + void (*free_privdata)(void *); + void (*async_read)(struct redisAsyncContext *); + void (*async_write)(struct redisAsyncContext *); + int (*read)(struct redisContext *, char *, size_t); + int (*write)(struct redisContext *); +} redisContextFuncs; + /* Context for a connection to Redis */ typedef struct redisContext { + const redisContextFuncs *funcs; /* Function table */ + int err; /* Error flags, 0 when there is no error */ char errstr[128]; /* String representation of error when applicable */ - int fd; + redisFD fd; int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ @@ -139,8 +230,12 @@ typedef struct redisContext { /* For non-blocking connect */ struct sockadr *saddr; size_t addrlen; + + /* Additional private data for hiredis addons such as SSL */ + void *privdata; } redisContext; +redisContext *redisConnectWithOptions(const redisOptions *options); redisContext *redisConnect(const char *ip, int port); redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); @@ -151,7 +246,7 @@ redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, redisContext *redisConnectUnix(const char *path); redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); +redisContext *redisConnectFd(redisFD fd); /** * Reconnect the given context using the saved information. @@ -167,7 +262,7 @@ int redisReconnect(redisContext *c); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); +redisFD redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in new file mode 100644 index 000000000..140b040f1 --- /dev/null +++ b/deps/hiredis/hiredis.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis +Description: Minimalistic C client library for Redis. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lhiredis +Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64 diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h new file mode 100644 index 000000000..f844f9548 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.h @@ -0,0 +1,53 @@ + +/* + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_SSL_H +#define __HIREDIS_SSL_H + +/* This is the underlying struct for SSL in ssl.h, which is not included to + * keep build dependencies short here. + */ +struct ssl_st; + +/** + * Secure the connection using SSL. This should be done before any command is + * executed on the connection. + */ +int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, + const char *keypath, const char *servername); + +/** + * Initiate SSL/TLS negotiation on a provided context. + */ + +int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); + +#endif /* __HIREDIS_SSL_H */ diff --git a/deps/hiredis/hiredis_ssl.pc.in b/deps/hiredis/hiredis_ssl.pc.in new file mode 100644 index 000000000..588a978a5 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis_ssl +Description: SSL Support for hiredis. +Version: @PROJECT_VERSION@ +Requires: hiredis +Libs: -L${libdir} -lhiredis_ssl +Libs.private: -lssl -lcrypto diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index a4b3abc6d..e5f40b0a4 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -34,36 +34,64 @@ #include "fmacros.h" #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #include #include #include "net.h" #include "sds.h" +#include "sockcompat.h" +#include "win32.h" /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { +void redisNetClose(redisContext *c) { + if (c && c->fd != REDIS_INVALID_FD) { close(c->fd); - c->fd = -1; + c->fd = REDIS_INVALID_FD; } } +int redisNetRead(redisContext *c, char *buf, size_t bufcap) { + int nread = recv(c->fd, buf, bufcap, 0); + if (nread == -1) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + return 0; + } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) { + /* especially in windows */ + __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout"); + return -1; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + return nread; + } +} + +int redisNetWrite(redisContext *c) { + int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); + if (nwritten < 0) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return nwritten; +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { int errorno = errno; /* snprintf() may change errno */ char buf[128] = { 0 }; @@ -79,15 +107,15 @@ static int redisSetReuseAddr(redisContext *c) { int on = 1; if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; } static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + redisFD s; + if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } @@ -101,6 +129,7 @@ static int redisCreateSocket(redisContext *c, int type) { } static int redisSetBlocking(redisContext *c, int blocking) { +#ifndef _WIN32 int flags; /* Set the socket nonblocking. @@ -108,7 +137,7 @@ static int redisSetBlocking(redisContext *c, int blocking) { * interrupted by a signal. */ if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -119,15 +148,23 @@ static int redisSetBlocking(redisContext *c, int blocking) { if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } +#else + u_long mode = blocking ? 0 : 1; + if (ioctl(c->fd, FIONBIO, &mode) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)"); + redisNetClose(c); + return REDIS_ERR; + } +#endif /* _WIN32 */ return REDIS_OK; } int redisKeepAlive(redisContext *c, int interval) { int val = 1; - int fd = c->fd; + redisFD fd = c->fd; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); @@ -170,7 +207,7 @@ static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; @@ -212,12 +249,12 @@ static int redisContextWaitReady(redisContext *c, long msec) { if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -230,7 +267,7 @@ static int redisContextWaitReady(redisContext *c, long msec) { } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -277,11 +314,18 @@ int redisCheckSocketError(redisContext *c) { } int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + const void *to_ptr = &tv; + size_t to_sz = sizeof(tv); +#ifdef _WIN32 + DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; + to_ptr = &timeout_msec; + to_sz = sizeof(timeout_msec); +#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); return REDIS_ERR; } @@ -291,7 +335,8 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { - int s, rv, n; + redisFD s; + int rv, n; char _port[6]; /* strlen("65535"); */ struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); @@ -360,7 +405,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } for (p = servinfo; p != NULL; p = p->ai_next) { addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD) continue; c->fd = s; @@ -401,16 +446,14 @@ addrretry: } /* For repeat connection */ - if (c->saddr) { - free(c->saddr); - } + free(c->saddr); c->saddr = malloc(p->ai_addrlen); memcpy(c->saddr, p->ai_addr, p->ai_addrlen); c->addrlen = p->ai_addrlen; if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); + redisNetClose(c); continue; } else if (errno == EINPROGRESS) { if (blocking) { @@ -424,7 +467,7 @@ addrretry: if (++reuses >= REDIS_CONNECT_RETRIES) { goto error; } else { - redisContextCloseFd(c); + redisNetClose(c); goto addrretry; } } else { @@ -471,8 +514,9 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, } int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { +#ifndef _WIN32 int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; + struct sockaddr_un *sa; long timeout_msec = -1; if (redisCreateSocket(c,AF_UNIX) < 0) @@ -499,9 +543,11 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa.sun_family = AF_UNIX; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + c->addrlen = sizeof(struct sockaddr_un); + sa->sun_family = AF_UNIX; + strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); + if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { @@ -516,4 +562,10 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time c->flags |= REDIS_CONNECTED; return REDIS_OK; +#else + /* We currently do not support Unix sockets for Windows. */ + /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */ + errno = EPROTONOSUPPORT; + return REDIS_ERR; +#endif /* _WIN32 */ } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index a11594e68..a4393c06b 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -37,6 +37,10 @@ #include "hiredis.h" +void redisNetClose(redisContext *c); +int redisNetRead(redisContext *c, char *buf, size_t bufcap); +int redisNetWrite(redisContext *c); + int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index cc0f3cc72..b9853ea9a 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -31,10 +31,10 @@ #include "fmacros.h" #include -#include #include #ifndef _MSC_VER #include +#include #endif #include #include @@ -44,6 +44,7 @@ #include "read.h" #include "sds.h" +#include "win32.h" static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -294,9 +295,9 @@ static int processLineItem(redisReader *r) { buf[len] = '\0'; if (strcasecmp(buf,",inf") == 0) { - d = 1.0/0.0; /* Positive infinite. */ + d = INFINITY; /* Positive infinite. */ } else if (strcasecmp(buf,",-inf") == 0) { - d = -1.0/0.0; /* Nevative infinite. */ + d = -INFINITY; /* Nevative infinite. */ } else { d = strtod((char*)buf,&eptr); if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { @@ -379,10 +380,18 @@ static int processBulkItem(redisReader *r) { /* Only continue when the buffer contains the entire bulk item. */ bytelen += len+2; /* include \r\n */ if (r->pos+bytelen <= r->len) { + if ((cur->type == REDIS_REPLY_VERB && len < 4) || + (cur->type == REDIS_REPLY_VERB && s[5] != ':')) + { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Verbatim string 4 bytes of content type are " + "missing or incorrectly encoded."); + return REDIS_ERR; + } if (r->fn && r->fn->createString) obj = r->fn->createString(cur,s+2,len); else - obj = (void*)REDIS_REPLY_STRING; + obj = (void*)(long)cur->type; success = 1; } } @@ -430,7 +439,7 @@ static int processAggregateItem(redisReader *r) { root = (r->ridx == 0); - if (elements < -1 || elements > INT_MAX) { + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "Multi-bulk length out of range"); return REDIS_ERR; @@ -523,6 +532,9 @@ static int processItem(redisReader *r) { case '#': cur->type = REDIS_REPLY_BOOL; break; + case '=': + cur->type = REDIS_REPLY_VERB; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -543,6 +555,7 @@ static int processItem(redisReader *r) { case REDIS_REPLY_BOOL: return processLineItem(r); case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: return processBulkItem(r); case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: @@ -657,8 +670,11 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Emit a reply when there is one. */ if (r->ridx == -1) { - if (reply != NULL) + if (reply != NULL) { *reply = r->reply; + } else if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + } r->reply = NULL; } return REDIS_OK; diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h index f3d075843..58105312a 100644 --- a/deps/hiredis/read.h +++ b/deps/hiredis/read.h @@ -45,6 +45,7 @@ #define REDIS_ERR_EOF 3 /* End of file */ #define REDIS_ERR_PROTOCOL 4 /* Protocol error */ #define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_TIMEOUT 6 /* Timed out */ #define REDIS_ERR_OTHER 2 /* Everything else... */ #define REDIS_REPLY_STRING 1 @@ -55,12 +56,12 @@ #define REDIS_REPLY_ERROR 6 #define REDIS_REPLY_DOUBLE 7 #define REDIS_REPLY_BOOL 8 -#define REDIS_REPLY_VERB 9 #define REDIS_REPLY_MAP 9 #define REDIS_REPLY_SET 10 #define REDIS_REPLY_ATTR 11 #define REDIS_REPLY_PUSH 12 #define REDIS_REPLY_BIGNUM 13 +#define REDIS_REPLY_VERB 14 #define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ @@ -79,7 +80,7 @@ typedef struct redisReadTask { typedef struct redisReplyObjectFunctions { void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); + void *(*createArray)(const redisReadTask*, size_t); void *(*createInteger)(const redisReadTask*, long long); void *(*createDouble)(const redisReadTask*, double, char*, size_t); void *(*createNil)(const redisReadTask*); diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 44777b10c..6cf75841c 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1035,7 +1035,7 @@ sds *sdssplitargs(const char *line, int *argc) { s_free(vector); return NULL; } - + vector = new_vector; vector[*argc] = current; (*argc)++; diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index bb99f7c52..0ce33b1fc 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -34,6 +34,9 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) +#ifdef _MSC_VER +#define __attribute__(x) +#endif #include #include @@ -143,20 +146,20 @@ static inline void sdssetlen(sds s, size_t newlen) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len = newlen; + SDS_HDR(8,s)->len = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len = newlen; + SDS_HDR(16,s)->len = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len = newlen; + SDS_HDR(32,s)->len = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len = newlen; + SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } @@ -167,21 +170,21 @@ static inline void sdsinclen(sds s, size_t inc) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len += inc; + SDS_HDR(8,s)->len += (uint8_t)inc; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len += inc; + SDS_HDR(16,s)->len += (uint16_t)inc; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len += inc; + SDS_HDR(32,s)->len += (uint32_t)inc; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len += inc; + SDS_HDR(64,s)->len += (uint64_t)inc; break; } } @@ -211,16 +214,16 @@ static inline void sdssetalloc(sds s, size_t newlen) { /* Nothing to do, this type has no total allocation info. */ break; case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = newlen; + SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = newlen; + SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = newlen; + SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = newlen; + SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c new file mode 100644 index 000000000..4cc2f414f --- /dev/null +++ b/deps/hiredis/sockcompat.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDIS_SOCKCOMPAT_IMPLEMENTATION +#include "sockcompat.h" + +#ifdef _WIN32 +static int _wsaErrorToErrno(int err) { + switch (err) { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEINPROGRESS: + return EINPROGRESS; + case WSAEALREADY: + return EALREADY; + case WSAENOTSOCK: + return ENOTSOCK; + case WSAEDESTADDRREQ: + return EDESTADDRREQ; + case WSAEMSGSIZE: + return EMSGSIZE; + case WSAEPROTOTYPE: + return EPROTOTYPE; + case WSAENOPROTOOPT: + return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: + return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: + return EOPNOTSUPP; + case WSAEAFNOSUPPORT: + return EAFNOSUPPORT; + case WSAEADDRINUSE: + return EADDRINUSE; + case WSAEADDRNOTAVAIL: + return EADDRNOTAVAIL; + case WSAENETDOWN: + return ENETDOWN; + case WSAENETUNREACH: + return ENETUNREACH; + case WSAENETRESET: + return ENETRESET; + case WSAECONNABORTED: + return ECONNABORTED; + case WSAECONNRESET: + return ECONNRESET; + case WSAENOBUFS: + return ENOBUFS; + case WSAEISCONN: + return EISCONN; + case WSAENOTCONN: + return ENOTCONN; + case WSAETIMEDOUT: + return ETIMEDOUT; + case WSAECONNREFUSED: + return ECONNREFUSED; + case WSAELOOP: + return ELOOP; + case WSAENAMETOOLONG: + return ENAMETOOLONG; + case WSAEHOSTUNREACH: + return EHOSTUNREACH; + case WSAENOTEMPTY: + return ENOTEMPTY; + default: + /* We just return a generic I/O error if we could not find a relevant error. */ + return EIO; + } +} + +static void _updateErrno(int success) { + errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError()); +} + +static int _initWinsock() { + static int s_initialized = 0; + if (!s_initialized) { + static WSADATA wsadata; + int err = WSAStartup(MAKEWORD(2,2), &wsadata); + if (err != 0) { + errno = _wsaErrorToErrno(err); + return 0; + } + s_initialized = 1; + } + return 1; +} + +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return EAI_FAIL; + } + + switch (getaddrinfo(node, service, hints, res)) { + case 0: return 0; + case WSATRY_AGAIN: return EAI_AGAIN; + case WSAEINVAL: return EAI_BADFLAGS; + case WSAEAFNOSUPPORT: return EAI_FAMILY; + case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY; + case WSAHOST_NOT_FOUND: return EAI_NONAME; + case WSATYPE_NOT_FOUND: return EAI_SERVICE; + case WSAESOCKTNOSUPPORT: return EAI_SOCKTYPE; + default: return EAI_FAIL; /* Including WSANO_RECOVERY */ + } +} + +const char *win32_gai_strerror(int errcode) { + switch (errcode) { + case 0: errcode = 0; break; + case EAI_AGAIN: errcode = WSATRY_AGAIN; break; + case EAI_BADFLAGS: errcode = WSAEINVAL; break; + case EAI_FAMILY: errcode = WSAEAFNOSUPPORT; break; + case EAI_MEMORY: errcode = WSA_NOT_ENOUGH_MEMORY; break; + case EAI_NONAME: errcode = WSAHOST_NOT_FOUND; break; + case EAI_SERVICE: errcode = WSATYPE_NOT_FOUND; break; + case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT; break; + default: errcode = WSANO_RECOVERY; break; /* Including EAI_FAIL */ + } + return gai_strerror(errcode); +} + +void win32_freeaddrinfo(struct addrinfo *res) { + freeaddrinfo(res); +} + +SOCKET win32_socket(int domain, int type, int protocol) { + SOCKET s; + + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return INVALID_SOCKET; + } + + _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET); + return s; +} + +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) { + int ret = ioctlsocket(fd, (long)request, argp); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = bind(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = connect(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + + /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as + * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX + * logic consistent. */ + if (errno == EWOULDBLOCK) { + errno = EINPROGRESS; + } + + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + if (*optlen >= sizeof (struct timeval)) { + struct timeval *tv = optval; + DWORD timeout = 0; + socklen_t dwlen = 0; + ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen); + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout * 1000) % 1000000; + } else { + ret = WSAEFAULT; + } + *optlen = sizeof (struct timeval); + } else { + ret = getsockopt(sockfd, level, optname, (char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + struct timeval *tv = optval; + DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; + ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); + } else { + ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_close(SOCKET fd) { + int ret = closesocket(fd); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) { + int ret = recv(sockfd, (char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) { + int ret = send(sockfd, (const char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) { + int ret = WSAPoll(fds, nfds, timeout); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} +#endif /* _WIN32 */ diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h new file mode 100644 index 000000000..56006c163 --- /dev/null +++ b/deps/hiredis/sockcompat.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SOCKCOMPAT_H +#define __SOCKCOMPAT_H + +#ifndef _WIN32 +/* For POSIX systems we use the standard BSD socket API. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#else +/* For Windows we use winsock. */ +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */ +#include +#include +#include + +#ifdef _MSC_VER +typedef signed long ssize_t; +#endif + +/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res); +const char *win32_gai_strerror(int errcode); +void win32_freeaddrinfo(struct addrinfo *res); +SOCKET win32_socket(int domain, int type, int protocol); +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp); +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen); +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen); +int win32_close(SOCKET fd); +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags); +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags); +typedef ULONG nfds_t; +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout); + +#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION +#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res) +#undef gai_strerror +#define gai_strerror(errcode) win32_gai_strerror(errcode) +#define freeaddrinfo(res) win32_freeaddrinfo(res) +#define socket(domain, type, protocol) win32_socket(domain, type, protocol) +#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp) +#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen) +#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen) +#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen) +#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen) +#define close(fd) win32_close(fd) +#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags) +#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags) +#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout) +#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */ +#endif /* _WIN32 */ + +#endif /* __SOCKCOMPAT_H */ diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c new file mode 100644 index 000000000..78ab9e43e --- /dev/null +++ b/deps/hiredis/ssl.c @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "hiredis.h" +#include "async.h" + +#include +#include +#include +#include + +#include +#include + +#include "async_private.h" + +void __redisSetError(redisContext *c, int type, const char *str); + +/* The SSL context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSLContext { + /** + * OpenSSL SSL_CTX; It is optional and will not be set when using + * user-supplied SSL. + */ + SSL_CTX *ssl_ctx; + + /** + * OpenSSL SSL object. + */ + SSL *ssl; + + /** + * SSL_write() requires to be called again with the same arguments it was + * previously called with in the event of an SSL_read/SSL_write situation + */ + size_t lastLen; + + /** Whether the SSL layer requires read (possibly before a write) */ + int wantRead; + + /** + * Whether a write was requested prior to a read. If set, the write() + * should resume whenever a read takes place, if possible + */ + int pendingWrite; +} redisSSLContext; + +/* Forward declaration */ +redisContextFuncs redisContextSSLFuncs; + +#ifdef HIREDIS_SSL_TRACE +/** + * Callback used for debugging + */ +static void sslLogCallback(const SSL *ssl, int where, int ret) { + const char *retstr = ""; + int should_log = 1; + /* Ignore low-level SSL stuff */ + + if (where & SSL_CB_ALERT) { + should_log = 1; + } + if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { + should_log = 1; + } + if ((where & SSL_CB_EXIT) && ret == 0) { + should_log = 1; + } + + if (!should_log) { + return; + } + + retstr = SSL_alert_type_string(ret); + printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); + + if (where == SSL_CB_HANDSHAKE_DONE) { + printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); + } +} +#endif + +/** + * OpenSSL global initialization and locking handling callbacks. + * Note that this is only required for OpenSSL < 1.1.0. + */ + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HIREDIS_USE_CRYPTO_LOCKS +#endif + +#ifdef HIREDIS_USE_CRYPTO_LOCKS +typedef pthread_mutex_t sslLockType; +static void sslLockInit(sslLockType *l) { + pthread_mutex_init(l, NULL); +} +static void sslLockAcquire(sslLockType *l) { + pthread_mutex_lock(l); +} +static void sslLockRelease(sslLockType *l) { + pthread_mutex_unlock(l); +} +static pthread_mutex_t *ossl_locks; + +static void opensslDoLock(int mode, int lkid, const char *f, int line) { + sslLockType *l = ossl_locks + lkid; + + if (mode & CRYPTO_LOCK) { + sslLockAcquire(l); + } else { + sslLockRelease(l); + } + + (void)f; + (void)line; +} + +static void initOpensslLocks(void) { + unsigned ii, nlocks; + if (CRYPTO_get_locking_callback() != NULL) { + /* Someone already set the callback before us. Don't destroy it! */ + return; + } + nlocks = CRYPTO_num_locks(); + ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + for (ii = 0; ii < nlocks; ii++) { + sslLockInit(ossl_locks + ii); + } + CRYPTO_set_locking_callback(opensslDoLock); +} +#endif /* HIREDIS_USE_CRYPTO_LOCKS */ + +/** + * SSL Connection initialization. + */ + +static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { + if (c->privdata) { + __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); + return REDIS_ERR; + } + c->privdata = calloc(1, sizeof(redisSSLContext)); + + c->funcs = &redisContextSSLFuncs; + redisSSLContext *rssl = c->privdata; + + rssl->ssl_ctx = ssl_ctx; + rssl->ssl = ssl; + + SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_set_fd(rssl->ssl, c->fd); + SSL_set_connect_state(rssl->ssl); + + ERR_clear_error(); + int rv = SSL_connect(rssl->ssl); + if (rv == 1) { + return REDIS_OK; + } + + rv = SSL_get_error(rssl->ssl, rv); + if (((c->flags & REDIS_BLOCK) == 0) && + (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + return REDIS_OK; + } + + if (c->err == 0) { + char err[512]; + if (rv == SSL_ERROR_SYSCALL) + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno)); + else { + unsigned long e = ERR_peek_last_error(); + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s", + ERR_reason_error_string(e)); + } + __redisSetError(c, REDIS_ERR_IO, err); + } + return REDIS_ERR; +} + +int redisInitiateSSL(redisContext *c, SSL *ssl) { + return redisSSLConnect(c, NULL, ssl); +} + +int redisSecureConnection(redisContext *c, const char *capath, + const char *certpath, const char *keypath, const char *servername) { + + SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; + + /* Initialize global OpenSSL stuff */ + static int isInit = 0; + if (!isInit) { + isInit = 1; + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + } + + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); + goto error; + } + +#ifdef HIREDIS_SSL_TRACE + SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); +#endif + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); + goto error; + } + + if (capath) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); + goto error; + } + } + if (certpath) { + if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); + goto error; + } + } + + ssl = SSL_new(ssl_ctx); + if (!ssl) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); + goto error; + } + if (servername) { + if (!SSL_set_tlsext_host_name(ssl, servername)) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + goto error; + } + } + + return redisSSLConnect(c, ssl_ctx, ssl); + +error: + if (ssl) SSL_free(ssl); + if (ssl_ctx) SSL_CTX_free(ssl_ctx); + return REDIS_ERR; +} + +static int maybeCheckWant(redisSSLContext *rssl, int rv) { + /** + * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set + * and true is returned. False is returned otherwise + */ + if (rv == SSL_ERROR_WANT_READ) { + rssl->wantRead = 1; + return 1; + } else if (rv == SSL_ERROR_WANT_WRITE) { + rssl->pendingWrite = 1; + return 1; + } else { + return 0; + } +} + +/** + * Implementation of redisContextFuncs for SSL connections. + */ + +static void redisSSLFreeContext(void *privdata){ + redisSSLContext *rsc = privdata; + + if (!rsc) return; + if (rsc->ssl) { + SSL_free(rsc->ssl); + rsc->ssl = NULL; + } + if (rsc->ssl_ctx) { + SSL_CTX_free(rsc->ssl_ctx); + rsc->ssl_ctx = NULL; + } + free(rsc); +} + +static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSLContext *rssl = c->privdata; + + int nread = SSL_read(rssl->ssl, buf, bufcap); + if (nread > 0) { + return nread; + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + int err = SSL_get_error(rssl->ssl, nread); + if (c->flags & REDIS_BLOCK) { + /** + * In blocking mode, we should never end up in a situation where + * we get an error without it being an actual error, except + * in the case of EINTR, which can be spuriously received from + * debuggers or whatever. + */ + if (errno == EINTR) { + return 0; + } else { + const char *msg = NULL; + if (errno == EAGAIN) { + msg = "Resource temporarily unavailable"; + } + __redisSetError(c, REDIS_ERR_IO, msg); + return -1; + } + } + + /** + * We can very well get an EWOULDBLOCK/EAGAIN, however + */ + if (maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } +} + +static int redisSSLWrite(redisContext *c) { + redisSSLContext *rssl = c->privdata; + + size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + int rv = SSL_write(rssl->ssl, c->obuf, len); + + if (rv > 0) { + rssl->lastLen = 0; + } else if (rv < 0) { + rssl->lastLen = len; + + int err = SSL_get_error(rssl->ssl, rv); + if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return rv; +} + +static void redisSSLAsyncRead(redisAsyncContext *ac) { + int rv; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->wantRead = 0; + + if (rssl->pendingWrite) { + int done; + + /* This is probably just a write event */ + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } else if (!done) { + _EL_ADD_WRITE(ac); + } + } + + rv = redisBufferRead(c); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +static void redisSSLAsyncWrite(redisAsyncContext *ac) { + int rv, done = 0; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } + + if (!done) { + if (rssl->wantRead) { + /* Need to read-before-write */ + rssl->pendingWrite = 1; + _EL_DEL_WRITE(ac); + } else { + /* No extra reads needed, just need to write more */ + _EL_ADD_WRITE(ac); + } + } else { + /* Already done! */ + _EL_DEL_WRITE(ac); + } + + /* Always reschedule a read */ + _EL_ADD_READ(ac); +} + +redisContextFuncs redisContextSSLFuncs = { + .free_privdata = redisSSLFreeContext, + .async_read = redisSSLAsyncRead, + .async_write = redisSSLAsyncWrite, + .read = redisSSLRead, + .write = redisSSLWrite +}; + diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 79cff4308..8668e1856 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -13,12 +13,16 @@ #include #include "hiredis.h" +#ifdef HIREDIS_TEST_SSL +#include "hiredis_ssl.h" +#endif #include "net.h" enum connection_type { CONN_TCP, CONN_UNIX, - CONN_FD + CONN_FD, + CONN_SSL }; struct config { @@ -33,6 +37,14 @@ struct config { struct { const char *path; } unix_sock; + + struct { + const char *host; + int port; + const char *ca_cert; + const char *cert; + const char *key; + } ssl; }; /* The following lines make up our testing "framework" :) */ @@ -93,11 +105,27 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } +static void do_ssl_handshake(redisContext *c, struct config config) { +#ifdef HIREDIS_TEST_SSL + redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + if (c->err) { + printf("SSL error: %s\n", c->errstr); + redisFree(c); + exit(1); + } +#else + (void) c; + (void) config; +#endif +} + static redisContext *do_connect(struct config config) { redisContext *c = NULL; if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_SSL) { + c = redisConnect(config.ssl.host, config.ssl.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix_sock.path); } else if (config.type == CONN_FD) { @@ -121,9 +149,21 @@ static redisContext *do_connect(struct config config) { exit(1); } + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } + return select_database(c); } +static void do_reconnect(redisContext *c, struct config config) { + redisReconnect(c); + + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } +} + static void test_format_commands(void) { char *cmd; int len; @@ -360,7 +400,8 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); - test("Set error when array > INT_MAX: "); +#if LLONG_MAX > SIZE_MAX + test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29); ret = redisReaderGetReply(reader,&reply); @@ -369,7 +410,6 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); -#if LLONG_MAX > SIZE_MAX test("Set error when bulk > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28); @@ -434,22 +474,23 @@ static void test_free_null(void) { test_cond(reply == NULL); } +#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; struct addrinfo hints = {.ai_family = AF_INET}; struct addrinfo *ai_tmp = NULL; - const char *bad_domain = "idontexist.com"; - int rv = getaddrinfo(bad_domain, "6379", &hints, &ai_tmp); + int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp); if (rv != 0) { // Address does *not* exist test("Returns error when host cannot be resolved: "); // First see if this domain name *actually* resolves to NXDOMAIN - c = redisConnect("dontexist.com", 6379); + c = redisConnect(HIREDIS_BAD_DOMAIN, 6379); test_cond( c->err == REDIS_ERR_OTHER && (strcmp(c->errstr, "Name or service not known") == 0 || - strcmp(c->errstr, "Can't resolve: sadkfjaskfjsa.com") == 0 || + strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || + strcmp(c->errstr, "Name does not resolve") == 0 || strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || strcmp(c->errstr, "No address associated with hostname") == 0 || @@ -574,7 +615,8 @@ static void test_blocking_connection_timeouts(struct config config) { c = do_connect(config); test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); + redisAppendFormattedCommand(c, cmd, strlen(cmd)); + s = c->funcs->write(c); tv.tv_sec = 0; tv.tv_usec = 10000; redisSetTimeout(c, tv); @@ -583,7 +625,7 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -591,7 +633,7 @@ static void test_blocking_connection_timeouts(struct config config) { test("Reconnect properly uses owned parameters: "); config.tcp.host = "foo"; config.unix_sock.path = "foo"; - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -894,6 +936,23 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; +#ifdef HIREDIS_TEST_SSL + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { + argv++; argc--; + cfg.ssl.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) { + argv++; argc--; + cfg.ssl.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) { + argv++; argc--; + cfg.ssl.ca_cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) { + argv++; argc--; + cfg.ssl.cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) { + argv++; argc--; + cfg.ssl.key = argv[0]; +#endif } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -922,6 +981,20 @@ int main(int argc, char **argv) { test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); +#ifdef HIREDIS_TEST_SSL + if (cfg.ssl.port && cfg.ssl.host) { + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); + cfg.type = CONN_SSL; + + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + } +#endif + if (test_inherit_fd) { printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); cfg.type = CONN_FD; diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh new file mode 100755 index 000000000..2cab9e6fb --- /dev/null +++ b/deps/hiredis/test.sh @@ -0,0 +1,70 @@ +#!/bin/sh -ue + +REDIS_SERVER=${REDIS_SERVER:-redis-server} +REDIS_PORT=${REDIS_PORT:-56379} +REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} +TEST_SSL=${TEST_SSL:-0} +SSL_TEST_ARGS= + +tmpdir=$(mktemp -d) +PID_FILE=${tmpdir}/hiredis-test-redis.pid +SOCK_FILE=${tmpdir}/hiredis-test-redis.sock + +if [ "$TEST_SSL" = "1" ]; then + SSL_CA_CERT=${tmpdir}/ca.crt + SSL_CA_KEY=${tmpdir}/ca.key + SSL_CERT=${tmpdir}/redis.crt + SSL_KEY=${tmpdir}/redis.key + + openssl genrsa -out ${tmpdir}/ca.key 4096 + openssl req \ + -x509 -new -nodes -sha256 \ + -key ${SSL_CA_KEY} \ + -days 3650 \ + -subj '/CN=Hiredis Test CA' \ + -out ${SSL_CA_CERT} + openssl genrsa -out ${SSL_KEY} 2048 + openssl req \ + -new -sha256 \ + -key ${SSL_KEY} \ + -subj '/CN=Hiredis Test Cert' | \ + openssl x509 \ + -req -sha256 \ + -CA ${SSL_CA_CERT} \ + -CAkey ${SSL_CA_KEY} \ + -CAserial ${tmpdir}/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out ${SSL_CERT} + + SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}" +fi + +cleanup() { + set +e + kill $(cat ${PID_FILE}) + rm -rf ${tmpdir} +} +trap cleanup INT TERM EXIT + +cat > ${tmpdir}/redis.conf <> ${tmpdir}/redis.conf < /* for struct timeval */ + #ifndef inline #define inline __inline #endif +#ifndef strcasecmp +#define strcasecmp stricmp +#endif + +#ifndef strncasecmp +#define strncasecmp strnicmp +#endif + #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif @@ -37,6 +47,10 @@ __inline int c99_snprintf(char* str, size_t size, const char* format, ...) return count; } #endif +#endif /* _MSC_VER */ -#endif -#endif \ No newline at end of file +#ifdef _WIN32 +#define strerror_r(errno,buf,len) strerror_s(buf,len,errno) +#endif /* _WIN32 */ + +#endif /* _WIN32_HELPER_INCLUDE */ diff --git a/deps/jemalloc/src/background_thread.c b/deps/jemalloc/src/background_thread.c index 3517a3bb8..457669c9e 100644 --- a/deps/jemalloc/src/background_thread.c +++ b/deps/jemalloc/src/background_thread.c @@ -787,7 +787,13 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { nstime_init(&stats->run_interval, 0); for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; - malloc_mutex_lock(tsdn, &info->mtx); + if (malloc_mutex_trylock(tsdn, &info->mtx)) { + /* + * Each background thread run may take a long time; + * avoid waiting on the stats if the thread is active. + */ + continue; + } if (info->state != background_thread_stopped) { num_runs += info->tot_n_runs; nstime_add(&stats->run_interval, &info->tot_sleep_time); diff --git a/keydb.conf b/keydb.conf index 03087b0c0..896528241 100644 --- a/keydb.conf +++ b/keydb.conf @@ -129,6 +129,76 @@ timeout 0 # KeyDB default starting with Redis 3.2.1. tcp-keepalive 300 +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file redis.crt tls-key-file redis.key + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file redis.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# If TLS/SSL clients are required to authenticate using a client side +# certificate, use this directive. +# +# Note: this applies to all incoming clients, including replicas. +# +# tls-auth-clients yes + +# If TLS/SSL should be used when connecting as a replica to a master, enable +# this configuration directive: +# +# tls-replication yes + +# If TLS/SSL should be used for the Redis Cluster bus, enable this configuration +# directive. +# +# NOTE: If TLS/SSL is enabled for Cluster Bus, mutual authentication is always +# enforced. +# +# tls-cluster yes + +# Explicitly specify TLS versions to support. Allowed values are case insensitive +# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or +# "default" which is currently >= TLSv1.1. +# +# tls-protocols TLSv1.2 + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + ################################# GENERAL ##################################### # By default KeyDB does not run as a daemon. Use 'yes' if you need it. @@ -336,13 +406,11 @@ replica-read-only yes # Replication SYNC strategy: disk or socket. # -# ------------------------------------------------------- -# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY -# ------------------------------------------------------- +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. # -# New replicas and reconnecting replicas that are not able to continue the replication -# process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the replicas. # The transmission can happen in two different ways: # # 1) Disk-backed: The KeyDB master creates a new process that writes the RDB @@ -352,14 +420,14 @@ replica-read-only yes # RDB file to replica sockets, without touching the disk at all. # # With disk-backed replication, while the RDB file is generated, more replicas -# can be queued and served with the RDB file as soon as the current child producing -# the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new replicas arriving will be queued and a new transfer -# will start when the current one terminates. +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple replicas -# will arrive and the transfer can be parallelized. +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication # works better. @@ -370,16 +438,42 @@ repl-diskless-sync no # to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new replicas arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more replicas arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if your do what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# recived from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. # # repl-ping-replica-period 10 @@ -411,10 +505,10 @@ repl-diskless-sync-delay 5 repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# replica data when replicas are disconnected for some time, so that when a replica -# wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the replica missed while -# disconnected. +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. # # The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. @@ -436,13 +530,13 @@ repl-disable-tcp-nodelay no # # repl-backlog-ttl 3600 -# The replica priority is an integer number published by KeyDB in the INFO output. -# It is used by KeyDB Sentinel in order to select a replica to promote into a -# master if the master is no longer working correctly. +# The replica priority is an integer number published by KeyDB in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. # # A replica with a low priority number is considered better for promotion, so -# for instance if there are three replicas with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. # # However a special priority of 0 marks the replica as not able to perform the # role of master, so a replica with priority of 0 will never be selected by @@ -502,6 +596,39 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 +############################### KEYS TRACKING ################################# + +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# 16 millions of slots, what clients may have certain subsets of keys. In turn +# this is used in order to send invalidation messages to clients. Please +# to understand more about the feature check this page: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 10%, and once this limit is +# reached, Redis will start to evict caching slots in the invalidation table +# even if keys are not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum fill rate is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and all the 16 +# millions of caching slots can be used at the same time. In the "stats" +# INFO section, you can find information about the amount of caching slots +# used at every given moment. +# +# tracking-table-max-fill 10 + ################################## SECURITY ################################### # Warning: since KeyDB is pretty fast an outside user can try up to @@ -684,13 +811,13 @@ replica-priority 100 # maxmemory # MAXMEMORY POLICY: how KeyDB will select what to remove when maxmemory -# is reached. You can select among five behaviors: +# is reached. You can select one from the following behaviors: # -# volatile-lru -> Evict using approximated LRU among the keys with an expire set. +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. # allkeys-lru -> Evict any key using approximated LRU. -# volatile-lfu -> Evict using approximated LFU among the keys with an expire set. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. # allkeys-lfu -> Evict any key using approximated LFU. -# volatile-random -> Remove a random key among the ones with an expire set. +# volatile-random -> Remove a random key having an expire set. # allkeys-random -> Remove a random key, any key. # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) # noeviction -> Don't evict anything, just return an error on write operations. @@ -731,20 +858,37 @@ replica-priority 100 # DEL commands to the replica as keys evict in the master side. # # This behavior ensures that masters and replicas stay consistent, and is usually -# what you want, however if your replica is writable, or you want the replica to have -# a different memory setting, and you are sure all the writes performed to the -# replica are idempotent, then you may change this default (but be sure to understand -# what you are doing). +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). # # Note that since the replica by default does not evict, it may end using more # memory than the one set via maxmemory (there are certain buffers that may -# be larger on the replica, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your replicas and make sure they have enough -# memory to never hit a real out-of-memory condition before the master hits -# the configured maxmemory setting. +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. # # replica-ignore-maxmemory yes +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tollerate less already expired keys still present +# in the system. It's a tradeoff betweeen memory, CPU and latecy. +# +# active-expire-effort 1 + ############################# LAZY FREEING #################################### # KeyDB has two primitives to delete keys. One is called DEL and is a blocking @@ -1050,6 +1194,22 @@ lua-time-limit 5000 # # cluster-replica-no-failover no +# This option, when set to yes, allows nodes to serve read traffic while the +# the cluster is in a down state, as long as it believes it owns the slots. +# +# This is useful for two cases. The first case is for when an application +# doesn't require consistency of data during node failures or network partitions. +# One example of this is a cache, where as long as the node has the data it +# should be able to serve it. +# +# The second use case is for configurations that don't meet the recommended +# three shards but want to enable cluster mode and scale later. A +# master outage in a 1 or 2 shard configuration causes a read/write outage to the +# entire cluster without this option set, with it set there is only a write outage. +# Without a quorum of masters, slot ownership will not change automatically. +# +# cluster-allow-reads-when-down no + # In order to setup your cluster make sure to read the documentation # available at http://redis.io web site. @@ -1182,7 +1342,7 @@ notify-keyspace-events "" # KeyDB contains an implementation of the Gopher protocol, as specified in # the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). # -# The Gopher protocol was very popular in the late '90s. It is an alternative +# The Gopher protocol was very popular in the late '90s. It is an alternative # to the web, and the implementation both server and client side is so simple # that the KeyDB server has just 100 lines of code in order to implement this # support. @@ -1220,7 +1380,7 @@ notify-keyspace-events "" # to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. # Once a password is set: # -# 1. The Gopher server (when enabled, not by default) will kill serve +# 1. The Gopher server (when enabled, not by default) will still serve # content via Gopher. # 2. However other commands cannot be called before the client will # authenticate. @@ -1479,10 +1639,6 @@ rdb-save-incremental-fsync yes ########################### ACTIVE DEFRAGMENTATION ####################### # -# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested -# even in production and manually tested by multiple engineers for some -# time. -# # What is active defragmentation? # ------------------------------- # @@ -1522,7 +1678,7 @@ rdb-save-incremental-fsync yes # a good idea to leave the defaults untouched. # Enabled active defragmentation -# activedefrag yes +# activedefrag no # Minimum amount of fragmentation waste to start active defrag # active-defrag-ignore-bytes 100mb @@ -1533,11 +1689,13 @@ rdb-save-incremental-fsync yes # Maximum percentage of fragmentation at which we use maximum effort # active-defrag-threshold-upper 100 -# Minimal effort for defrag in CPU percentage -# active-defrag-cycle-min 5 +# Minimal effort for defrag in CPU percentage, to be used when the lower +# threshold is reached +# active-defrag-cycle-min 1 -# Maximal effort for defrag in CPU percentage -# active-defrag-cycle-max 75 +# Maximal effort for defrag in CPU percentage, to be used when the upper +# threshold is reached +# active-defrag-cycle-max 25 # Maximum number of set/hash/zset/list fields that will be processed from # the main dictionary scan diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb8..f6cc0a258 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,16 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl \ +--single unit/moduleapi/commandfilter \ +--single unit/moduleapi/fork \ +--single unit/moduleapi/testrdb \ +--single unit/moduleapi/infotest \ +--single unit/moduleapi/propagate \ +--single unit/moduleapi/hooks \ +--single unit/moduleapi/misc \ +--single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/scan \ +--single unit/moduleapi/datatype \ +--single unit/moduleapi/auth \ +"${@}" diff --git a/src/Makefile b/src/Makefile index a0e48831c..b89f69b53 100644 --- a/src/Makefile +++ b/src/Makefile @@ -21,7 +21,7 @@ NODEPS:=clean distclean # Default settings STD=-std=c11 -pedantic -DREDIS_STATIC='' -CXX_STD=-std=c++14 -pedantic -fno-rtti -D__STDC_FORMAT_MACROS +CXX_STD=-std=c++2a -pedantic -fno-rtti -D__STDC_FORMAT_MACROS ifneq (,$(findstring clang,$(CC))) ifneq (,$(findstring FreeBSD,$(uname_S))) STD+=-Wno-c11-extensions @@ -33,6 +33,7 @@ OPT=$(OPTIMIZATION) PREFIX?=/usr/local INSTALL_BIN=$(PREFIX)/bin INSTALL=install +PKG_CONFIG?=pkg-config # Default allocator defaults to Jemalloc if it's not an ARM MALLOC=libc @@ -108,6 +109,15 @@ FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) FINAL_LIBS=-lm DEBUG=-g -ggdb +# Linux ARM needs -latomic at linking time +ifneq (,$(filter aarch64 armv,$(uname_M))) + FINAL_LIBS+=-latomic +else +ifneq (,$(findstring armv,$(uname_M))) + FINAL_LIBS+=-latomic +endif +endif + ifeq ($(uname_S),SunOS) # SunOS ifneq ($(@@),32bit) @@ -126,6 +136,8 @@ else ifeq ($(uname_S),Darwin) # Darwin FINAL_LIBS+= -ldl + OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include + OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib else ifeq ($(uname_S),AIX) # AIX @@ -166,6 +178,30 @@ endif FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src +# Determine systemd support and/or build preference (defaulting to auto-detection) +BUILD_WITH_SYSTEMD=no +# If 'USE_SYSTEMD' in the environment is neither "no" nor "yes", try to +# auto-detect libsystemd's presence and link accordingly. +ifneq ($(USE_SYSTEMD),no) + LIBSYSTEMD_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libsystemd && echo $$?) +# If libsystemd cannot be detected, continue building without support for it +# (unless a later check tells us otherwise) +ifeq ($(LIBSYSTEMD_PKGCONFIG),0) + BUILD_WITH_SYSTEMD=yes +endif +endif +ifeq ($(USE_SYSTEMD),yes) +ifneq ($(LIBSYSTEMD_PKGCONFIG),0) +$(error USE_SYSTEMD is set to "$(USE_SYSTEMD)", but $(PKG_CONFIG) cannot find libsystemd) +endif +# Force building with libsystemd + BUILD_WITH_SYSTEMD=yes +endif +ifeq ($(BUILD_WITH_SYSTEMD),yes) + FINAL_LIBS+=$(shell $(PKG_CONFIG) --libs libsystemd) + FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD +endif + ifeq ($(MALLOC),tcmalloc) FINAL_CFLAGS+= -DUSE_TCMALLOC FINAL_CXXFLAGS+= -DUSE_TCMALLOC @@ -192,6 +228,12 @@ ifeq ($(MALLOC),memkind) FINAL_LIBS := ../deps/memkind/src/.libs/libmemkind.a -lnuma $(FINAL_LIBS) endif +ifeq ($(BUILD_TLS),yes) + FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS) + FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) + FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_CXX=$(QUIET_CC)$(CXX) $(FINAL_CXXFLAGS) KEYDB_AS=$(QUIET_CC) as --64 -g @@ -213,7 +255,7 @@ endif REDIS_SERVER_NAME=keydb-server REDIS_SENTINEL_NAME=keydb-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o $(ASM_OBJ) +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o $(ASM_OBJ) REDIS_CLI_NAME=keydb-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o new.o $(ASM_OBJ) REDIS_BENCHMARK_NAME=keydb-benchmark @@ -294,7 +336,6 @@ $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) - DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) -include $(DEP) @@ -302,10 +343,10 @@ DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ # building it should complete before building any other object. Instead of # depending on a single artifact, build all dependencies first. %.o: %.c .make-prerequisites - $(REDIS_CC) -MMD -c $< + $(REDIS_CC) -MMD -o $@ -c $< %.o: %.cpp .make-prerequisites - $(REDIS_CXX) -MMD -c $< + $(REDIS_CXX) -MMD -o $@ -c $< %.o: %.asm .make-prerequisites $(KEYDB_AS) $< -o $@ diff --git a/src/acl.cpp b/src/acl.cpp index 489f66a98..2c2fd18a9 100644 --- a/src/acl.cpp +++ b/src/acl.cpp @@ -28,6 +28,9 @@ */ #include "server.h" +extern "C" { +#include "sha256.h" +} #include /* ============================================================================= @@ -93,6 +96,9 @@ void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); +/* The length of the string representation of a hashed password. */ +#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 + /* ============================================================================= * Helper functions for the rest of the ACL implementation * ==========================================================================*/ @@ -139,6 +145,25 @@ int time_independent_strcmp(char *a, char *b) { return diff; /* If zero strings are the same. */ } +/* Given an SDS string, returns the SHA256 hex representation as a + * new SDS string. */ +sds ACLHashPassword(unsigned char *cleartext, size_t len) { + SHA256_CTX ctx; + unsigned char hash[SHA256_BLOCK_SIZE]; + char hex[HASH_PASSWORD_LEN]; + const char *cset = "0123456789abcdef"; + + sha256_init(&ctx); + sha256_update(&ctx,(unsigned char*)cleartext,len); + sha256_final(&ctx,hash); + + for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { + hex[j*2] = cset[((hash[j]&0xF0)>>4)]; + hex[j*2+1] = cset[(hash[j]&0xF)]; + } + return sdsnewlen(hex,HASH_PASSWORD_LEN); +} + /* ============================================================================= * Low level ACL API * ==========================================================================*/ @@ -160,12 +185,12 @@ int ACLListMatchSds(void *a, void *b) { return sdscmp((sds)a,(sds)b) == 0; } -/* Method to free list elements from ACL users password/ptterns lists. */ +/* Method to free list elements from ACL users password/patterns lists. */ void ACLListFreeSds(const void *item) { sdsfree((sds)item); } -/* Method to duplicate list elements from ACL users password/ptterns lists. */ +/* Method to duplicate list elements from ACL users password/patterns lists. */ void *ACLListDupSds(void *item) { return sdsdup((sds)item); } @@ -502,7 +527,7 @@ sds ACLDescribeUser(user *u) { listRewind(u->passwords,&li); while((ln = listNext(&li))) { sds thispass = (sds)listNodeValue(ln); - res = sdscatlen(res,">",1); + res = sdscatlen(res,"#",1); res = sdscatsds(res,thispass); res = sdscatlen(res," ",1); } @@ -629,7 +654,14 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * > Add this password to the list of valid password for the user. * For example >mypass will add "mypass" to the list. * This directive clears the "nopass" flag (see later). + * # Add this password hash to the list of valid hashes for + * the user. This is useful if you have previously computed + * the hash, and don't want to store it in plaintext. + * This directive clears the "nopass" flag (see later). * < Remove this password from the list of valid passwords. + * ! Remove this hashed password from the list of valid passwords. + * This is useful when you want to remove a password just by + * hash without knowing its plaintext version at all. * nopass All the set passwords of the user are removed, and the user * is flagged as requiring no password: it means that every * password will work against this user. If this directive is @@ -665,6 +697,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * EEXIST: You are adding a key pattern after "*" was already added. This is * almost surely an error on the user side. * ENODEV: The password you are trying to remove from the user does not exist. + * EBADMSG: The hash you are trying to add is not a valid hash. */ int ACLSetUser(user *u, const char *op, ssize_t oplen) { if (oplen == -1) oplen = strlen(op); @@ -700,14 +733,48 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"resetpass")) { u->flags &= ~USER_FLAG_NOPASS; listEmpty(u->passwords); - } else if (op[0] == '>') { - sds newpass = sdsnewlen(op+1,oplen-1); + } else if (op[0] == '>' || op[0] == '#') { + sds newpass; + if (op[0] == '>') { + newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (oplen != HASH_PASSWORD_LEN + 1) { + errno = EBADMSG; + return C_ERR; + } + + /* Password hashes can only be characters that represent + * hexadecimal values, which are numbers and lowercase + * characters 'a' through 'f'. + */ + for(int i = 1; i < HASH_PASSWORD_LEN + 1; i++) { + char c = op[i]; + if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) { + errno = EBADMSG; + return C_ERR; + } + } + newpass = sdsnewlen(op+1,oplen-1); + } + listNode *ln = listSearchKey(u->passwords,newpass); /* Avoid re-adding the same password multiple times. */ - if (ln == NULL) listAddNodeTail(u->passwords,newpass); + if (ln == NULL) + listAddNodeTail(u->passwords,newpass); + else + sdsfree(newpass); u->flags &= ~USER_FLAG_NOPASS; - } else if (op[0] == '<') { - sds delpass = sdsnewlen(op+1,oplen-1); + } else if (op[0] == '<' || op[0] == '!') { + sds delpass; + if (op[0] == '<') { + delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (oplen != HASH_PASSWORD_LEN + 1) { + errno = EBADMSG; + return C_ERR; + } + delpass = sdsnewlen(op+1,oplen-1); + } listNode *ln = listSearchKey(u->passwords,delpass); sdsfree(delpass); if (ln) { @@ -724,7 +791,10 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); /* Avoid re-adding the same pattern multiple times. */ - if (ln == NULL) listAddNodeTail(u->patterns,newpat); + if (ln == NULL) + listAddNodeTail(u->patterns,newpat); + else + sdsfree(newpat); u->flags &= ~USER_FLAG_ALLKEYS; } else if (op[0] == '+' && op[1] != '@') { if (strchr(op,'|') == NULL) { @@ -822,6 +892,9 @@ const char *ACLSetUserStringError(void) { else if (errno == ENODEV) errmsg = "The password you are trying to remove from the user does " "not exist"; + else if (errno == EBADMSG) + errmsg = "The password hash must be exactly 64 characters and contain " + "only lowercase hexadecimal characters"; return errmsg; } @@ -879,11 +952,15 @@ int ACLCheckUserCredentials(robj *username, robj *password) { listIter li; listNode *ln; listRewind(u->passwords,&li); + sds hashed = ACLHashPassword((unsigned char*)szFromObj(password),sdslen(szFromObj(password))); while((ln = listNext(&li))) { sds thispass = (sds)listNodeValue(ln); - if (!time_independent_strcmp(szFromObj(password), thispass)) + if (!time_independent_strcmp(hashed, thispass)) { + sdsfree(hashed); return C_OK; + } } + sdsfree(hashed); /* If we reached this point, no password matched. */ errno = EINVAL; @@ -900,6 +977,7 @@ int ACLAuthenticateUser(client *c, robj *username, robj *password) { if (ACLCheckUserCredentials(username,password) == C_OK) { c->authenticated = 1; c->puser = ACLGetUserByName((sds)ptrFromObj(username),sdslen((sds)ptrFromObj(username))); + moduleNotifyUserChanged(c); return C_OK; } else { return C_ERR; diff --git a/src/adlist.h b/src/adlist.h index 99b0bca7a..996a2d83f 100644 --- a/src/adlist.h +++ b/src/adlist.h @@ -70,7 +70,7 @@ typedef struct list { #define listSetMatchMethod(l,m) ((l)->match = (m)) #define listGetDupMethod(l) ((l)->dup) -#define listGetFree(l) ((l)->free) +#define listGetFreeMethod(l) ((l)->free) #define listGetMatchMethod(l) ((l)->match) /* Prototypes */ diff --git a/src/ae.cpp b/src/ae.cpp index 90c148510..b7b33e057 100644 --- a/src/ae.cpp +++ b/src/ae.cpp @@ -324,6 +324,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->maxfd = -1; eventLoop->beforesleep = NULL; eventLoop->aftersleep = NULL; + eventLoop->flags = 0; if (aeApiCreate(eventLoop) == -1) goto err; /* Events with mask == AE_NONE are not set. So let's initialize the * vector with it. */ @@ -357,6 +358,14 @@ int aeGetSetSize(aeEventLoop *eventLoop) { return eventLoop->setsize; } +/* Tells the next iteration/s of the event processing to set timeout of 0. */ +void aeSetDontWait(aeEventLoop *eventLoop, int noWait) { + if (noWait) + eventLoop->flags |= AE_DONT_WAIT; + else + eventLoop->flags &= ~AE_DONT_WAIT; +} + /* Resize the maximum set size of the event loop. * If the requested set size is smaller than the current set size, but * there is already a file descriptor in use that is >= the requested @@ -750,6 +759,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) } } + if (eventLoop->flags & AE_DONT_WAIT) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } + /* Call the multiplexing API, will return only on timeout or when * some event fires. */ numevents = aeApiPoll(eventLoop, tvp); diff --git a/src/ae.h b/src/ae.h index 14ccc3dc8..e2dbcb754 100644 --- a/src/ae.h +++ b/src/ae.h @@ -124,6 +124,7 @@ typedef struct aeEventLoop { int fdCmdWrite; int fdCmdRead; int cevents; + int flags; } aeEventLoop; /* Prototypes */ @@ -157,6 +158,7 @@ void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep, int flags); int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); +void aeSetDontWait(aeEventLoop *eventLoop, int noWait); void aeAcquireLock(); int aeTryAcquireLock(int fWeak); diff --git a/src/ae_epoll.cpp b/src/ae_epoll.cpp index 05638ebdc..452fc49ae 100644 --- a/src/ae_epoll.cpp +++ b/src/ae_epoll.cpp @@ -125,8 +125,8 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (e->events & EPOLLIN) mask |= AE_READABLE; if (e->events & EPOLLOUT) mask |= AE_WRITABLE; - if (e->events & EPOLLERR) mask |= AE_WRITABLE; - if (e->events & EPOLLHUP) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE|AE_READABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE|AE_READABLE; eventLoop->fired[j].fd = e->data.fd; eventLoop->fired[j].mask = mask; } diff --git a/src/aelocker.h b/src/aelocker.h index eca15f491..ef757d2d2 100644 --- a/src/aelocker.h +++ b/src/aelocker.h @@ -11,6 +11,11 @@ public: void arm(client *c) // if a client is passed, then the client is already locked { + if (m_fArmed) + return; + + serverAssertDebug(!GlobalLocksAcquired()); + if (c != nullptr) { serverAssert(!m_fArmed); diff --git a/src/anet.c b/src/anet.c index b32dc6d46..8b8b6ae33 100644 --- a/src/anet.c +++ b/src/anet.c @@ -193,6 +193,20 @@ int anetSendTimeout(char *err, int fd, long long ms) { return ANET_OK; } +/* Set the socket receive timeout (SO_RCVTIMEO socket option) to the specified + * number of milliseconds, or disable it if the 'ms' argument is zero. */ +int anetRecvTimeout(char *err, int fd, long long ms) { + struct timeval tv; + + tv.tv_sec = ms/1000; + tv.tv_usec = (ms%1000)*1000; + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) { + anetSetError(err, "setsockopt SO_RCVTIMEO: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + /* anetGenericResolve() is called by anetResolve() and anetResolveIP() to * do the actual work. It resolves the hostname "host" and set the string * representation of the IP address into the buffer pointed by "ipbuf". @@ -276,8 +290,8 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONBLOCK 1 #define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */ #define ANET_CONNECT_REUSEPORT 4 -static int anetTcpGenericConnect(char *err, char *addr, int port, - char *source_addr, int flags) +static int anetTcpGenericConnect(char *err, const char *addr, int port, + const char *source_addr, int flags) { int s = ANET_ERR, rv; char portstr[6]; /* strlen("65535") + 1; */ @@ -359,31 +373,31 @@ end: } } -int anetTcpConnect(char *err, char *addr, int port) +int anetTcpConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); } -int anetTcpNonBlockConnect(char *err, char *addr, int port) +int anetTcpNonBlockConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, - char *source_addr) +int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, + const char *source_addr) { return anetTcpGenericConnect(err,addr,port,source_addr, ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, - char *source_addr) +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, + const char *source_addr) { return anetTcpGenericConnect(err,addr,port,source_addr, ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING); } -int anetUnixGenericConnect(char *err, char *path, int flags) +int anetUnixGenericConnect(char *err, const char *path, int flags) { int s; struct sockaddr_un sa; @@ -411,12 +425,12 @@ int anetUnixGenericConnect(char *err, char *path, int flags) return s; } -int anetUnixConnect(char *err, char *path) +int anetUnixConnect(char *err, const char *path) { return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); } -int anetUnixNonBlockConnect(char *err, char *path) +int anetUnixNonBlockConnect(char *err, const char *path) { return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); } diff --git a/src/anet.h b/src/anet.h index 06791ee1c..b31a38ad6 100644 --- a/src/anet.h +++ b/src/anet.h @@ -53,12 +53,12 @@ extern "C" { #undef ip_len #endif -int anetTcpConnect(char *err, char *addr, int port); -int anetTcpNonBlockConnect(char *err, char *addr, int port); -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, char *source_addr); -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, char *source_addr); -int anetUnixConnect(char *err, char *path); -int anetUnixNonBlockConnect(char *err, char *path); +int anetTcpConnect(char *err, const char *addr, int port); +int anetTcpNonBlockConnect(char *err, const char *addr, int port); +int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetUnixConnect(char *err, const char *path); +int anetUnixNonBlockConnect(char *err, const char *path); int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); @@ -74,6 +74,7 @@ int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); int anetSendTimeout(char *err, int fd, long long ms); +int anetRecvTimeout(char *err, int fd, long long ms); int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); int anetKeepAlive(char *err, int fd, int interval); int anetSockName(int fd, char *ip, size_t ip_len, int *port); diff --git a/src/aof.cpp b/src/aof.cpp index b82be9a34..8db2d4af2 100644 --- a/src/aof.cpp +++ b/src/aof.cpp @@ -271,9 +271,9 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (g_pserver->rdb_child_pid != -1) { + if (hasActiveChildProcess() && g_pserver->aof_child_pid == -1) { g_pserver->aof_rewrite_scheduled = 1; - serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); + serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not @@ -310,9 +310,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) { nwritten = write(fd, buf, len); if (nwritten < 0) { - if (errno == EINTR) { - continue; - } + if (errno == EINTR) continue; return totwritten ? totwritten : -1; } @@ -394,6 +392,10 @@ void flushAppendOnlyFile(int force) { * there is much to do about the whole server stopping for power problems * or alike */ + if (g_pserver->aof_flush_sleep && sdslen(g_pserver->aof_buf)) { + usleep(g_pserver->aof_flush_sleep); + } + latencyStartMonitor(latency); nwritten = aofWrite(g_pserver->aof_fd,g_pserver->aof_buf,sdslen(g_pserver->aof_buf)); latencyEndMonitor(latency); @@ -404,7 +406,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1) { + } else if (hasActiveChildProcess()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -500,9 +502,8 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (g_pserver->aof_no_fsync_on_rewrite && - (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1)) - return; + if (g_pserver->aof_no_fsync_on_rewrite && hasActiveChildProcess()) + return; /* Perform the fsync if needed. */ if (g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) { @@ -658,11 +659,12 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a /* In Redis commands are always executed in the context of a client, so in * order to load the append only file we need to create a fake client. */ -client *createFakeClient(void) { - client *c = (client*)zmalloc(sizeof(*c), MALLOC_LOCAL); +struct client *createAOFClient(void) { + struct client *c =(client*) zmalloc(sizeof(*c), MALLOC_LOCAL); selectDb(c,0); - c->fd = -1; + c->id = CLIENT_ID_AOF; /* So modules can identify it's the AOF client. */ + c->conn = NULL; c->iel = IDX_EVENT_LOOP_MAIN; c->name = NULL; c->querybuf = sdsempty(); @@ -742,8 +744,8 @@ int loadAppendOnlyFile(char *filename) { * to the same file we're about to read. */ g_pserver->aof_state = AOF_OFF; - fakeClient = createFakeClient(); - startLoading(fp); + fakeClient = createAOFClient(); + startLoadingFile(fp, filename, RDBFLAGS_AOF_PREAMBLE); /* Check if this AOF file has an RDB preamble. In that case we need to * load the RDB file and later continue loading the AOF tail. */ @@ -754,12 +756,12 @@ int loadAppendOnlyFile(char *filename) { } else { /* RDB preamble. Pass loading the RDB functions. */ rio rdb; + rdbSaveInfo rsi = RDB_SAVE_INFO_INIT; serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; rioInitWithFile(&rdb,fp); - rdbSaveInfo rsi = RDB_SAVE_INFO_INIT; - if (rdbLoadRio(&rdb,&rsi,1) != C_OK) { + if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,&rsi) != C_OK) { serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file, AOF loading aborted"); goto readerr; } else { @@ -780,6 +782,7 @@ int loadAppendOnlyFile(char *filename) { if (!(loops++ % 1000)) { loadingProgress(ftello(fp)); processEventsWhileBlocked(serverTL - g_pserver->rgthreadvar); + processModuleLoadingProgressEvent(1); } if (fgets(buf,sizeof(buf),fp) == NULL) { @@ -793,18 +796,24 @@ int loadAppendOnlyFile(char *filename) { argc = atoi(buf+1); if (argc < 1) goto fmterr; + /* Load the next command in the AOF as our fake client + * argv. */ argv = (robj**)zmalloc(sizeof(robj*)*argc, MALLOC_LOCAL); fakeClient->argc = argc; fakeClient->argv = argv; for (j = 0; j < argc; j++) { - if (fgets(buf,sizeof(buf),fp) == NULL) { + /* Parse the argument len. */ + if (fgets(buf,sizeof(buf),fp) == NULL || + buf[0] != '$') + { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); goto readerr; } - if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); + + /* Read it into a string object. */ argsds = sdsnewlen(SDS_NOINIT,len); if (len && fread(argsds,len,1,fp) == 0) { sdsfree(argsds); @@ -813,10 +822,12 @@ int loadAppendOnlyFile(char *filename) { goto readerr; } argv[j] = createObject(OBJ_STRING,argsds); + + /* Discard CRLF. */ if (fread(buf,2,1,fp) == 0) { fakeClient->argc = j+1; /* Free up to j. */ freeFakeClientArgv(fakeClient); - goto readerr; /* discard CRLF */ + goto readerr; } } @@ -853,6 +864,8 @@ int loadAppendOnlyFile(char *filename) { freeFakeClientArgv(fakeClient); fakeClient->cmd = NULL; if (g_pserver->aof_load_truncated) valid_up_to = ftello(fp); + if (g_pserver->key_load_delay) + usleep(g_pserver->key_load_delay); } /* This point can only be reached when EOF is reached without errors. @@ -870,7 +883,7 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ fclose(fp); freeFakeClient(fakeClient); g_pserver->aof_state = old_aof_state; - stopLoading(); + stopLoading(1); aofUpdateCurrentSize(); g_pserver->aof_rewrite_base_size = g_pserver->aof_current_size; g_pserver->aof_fsync_offset = g_pserver->aof_current_size; @@ -879,6 +892,7 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ if (!feof(fp)) { if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); exit(1); } @@ -909,11 +923,13 @@ uxeof: /* Unexpected AOF end of file. */ } } if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ - serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./keydb-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the g_pserver->"); + fclose(fp); + serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./keydb-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); fmterr: /* Format error. */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./keydb-check-aof --fix "); exit(1); } @@ -1421,9 +1437,11 @@ int rewriteAppendOnlyFile(char *filename) { if (g_pserver->aof_rewrite_incremental_fsync) rioSetAutoSync(&aof,REDIS_AUTOSYNC_BYTES); + startSaving(RDBFLAGS_AOF_PREAMBLE); + if (g_pserver->aof_use_rdb_preamble) { int error; - if (rdbSaveRio(&aof,&error,RDB_SAVE_AOF_PREAMBLE,NULL) == C_ERR) { + if (rdbSaveRio(&aof,&error,RDBFLAGS_AOF_PREAMBLE,NULL) == C_ERR) { errno = error; goto werr; } @@ -1485,15 +1503,18 @@ int rewriteAppendOnlyFile(char *filename) { if (rename(tmpfile,filename) == -1) { serverLog(LL_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno)); unlink(tmpfile); + stopSaving(0); return C_ERR; } serverLog(LL_NOTICE,"SYNC append only file rewrite performed"); + stopSaving(1); return C_OK; werr: serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); fclose(fp); unlink(tmpfile); + stopSaving(0); return C_ERR; } @@ -1599,39 +1620,24 @@ void aofClosePipes(void) { */ int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - long long start; - if (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { char tmpfile[256]; /* Child */ - closeListeningSockets(0); redisSetProcTitle("keydb-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "AOF rewrite: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - g_pserver->child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_AOF); + sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); } } else { /* Parent */ - g_pserver->stat_fork_time = ustime()-start; - g_pserver->stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / g_pserver->stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",g_pserver->stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); serverLog(LL_WARNING, @@ -1645,7 +1651,6 @@ int rewriteAppendOnlyFileBackground(void) { g_pserver->aof_rewrite_scheduled = 0; g_pserver->aof_rewrite_time_start = time(NULL); g_pserver->aof_child_pid = childpid; - updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences * accumulated by the parent into g_pserver->aof_rewrite_buf will start @@ -1660,13 +1665,14 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (g_pserver->aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (g_pserver->rdb_child_pid != -1) { + } else if (hasActiveChildProcess()) { g_pserver->aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { addReplyStatus(c,"Background append only file rewriting started"); } else { - addReply(c,shared.err); + addReplyError(c,"Can't execute an AOF background rewriting. " + "Please check the server logs for more information."); } } @@ -1805,6 +1811,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { g_pserver->aof_selected_db = -1; /* Make sure SELECT is re-issued */ aofUpdateCurrentSize(); g_pserver->aof_rewrite_base_size = g_pserver->aof_current_size; + g_pserver->aof_fsync_offset = g_pserver->aof_current_size; /* Clear regular AOF buffer since its contents was just written to * the new AOF from the background rewrite buffer. */ diff --git a/src/bio.h b/src/bio.h index 5349005b1..bf350f9de 100644 --- a/src/bio.h +++ b/src/bio.h @@ -1,3 +1,4 @@ +#pragma once /* * Copyright (c) 2009-2012, Salvatore Sanfilippo * All rights reserved. diff --git a/src/blocked.cpp b/src/blocked.cpp index 19d7f2fb6..a466c9c64 100644 --- a/src/blocked.cpp +++ b/src/blocked.cpp @@ -92,7 +92,7 @@ int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int } if (tval < 0) { - addReplyError(c,"timeout is negative"); + addReplyErrorAsync(c,"timeout is negative"); return C_ERR; } @@ -189,6 +189,7 @@ void unblockClient(client *c) { } else if (c->btype == BLOCKED_WAIT) { unblockClientWaitingReplicas(c); } else if (c->btype == BLOCKED_MODULE) { + if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); unblockClientFromModule(c); } else { serverPanic("Unknown btype in unblockClient()."); @@ -209,9 +210,9 @@ void replyToBlockedClientTimedOut(client *c) { if (c->btype == BLOCKED_LIST || c->btype == BLOCKED_ZSET || c->btype == BLOCKED_STREAM) { - addReplyNullArray(c); + addReplyNullArrayAsync(c); } else if (c->btype == BLOCKED_WAIT) { - addReplyLongLong(c,replicationCountAcksByOffset(c->bpop.reploffset)); + addReplyLongLongAsync(c,replicationCountAcksByOffset(c->bpop.reploffset)); } else if (c->btype == BLOCKED_MODULE) { moduleBlockedClientTimedOut(c); } else { @@ -247,6 +248,253 @@ void disconnectAllBlockedClients(void) { } } +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a list key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnListKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = (list*)dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = (client*)clientnode->value; + std::unique_locklock)> lock(receiver->lock); + + if (receiver->btype != BLOCKED_LIST) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + continue; + } + + robj *dstkey = receiver->bpop.target; + int where = (receiver->lastcmd && + receiver->lastcmd->proc == blpopCommand) ? + LIST_HEAD : LIST_TAIL; + robj *value = listTypePop(o,where); + + if (value) { + /* Protect receiver->bpop.target, that will be + * freed by the next unblockClient() + * call. */ + if (dstkey) incrRefCount(dstkey); + unblockClient(receiver); + + if (serveClientBlockedOnList(receiver, + rl->key,dstkey,rl->db,value, + where) == C_ERR) + { + /* If we failed serving the client we need + * to also undo the POP operation. */ + listTypePush(o,value,where); + } + + if (dstkey) decrRefCount(dstkey); + decrRefCount(value); + } else { + break; + } + } + } + + if (listTypeLength(o) == 0) { + dbDelete(rl->db,rl->key); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); + } + /* We don't call signalModifiedKey() as it was already called + * when an element was pushed on the list. */ +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a sorted set key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = (list*)dictGetVal(de); + int numclients = listLength(clients); + unsigned long zcard = zsetLength(o); + + while(numclients-- && zcard) { + listNode *clientnode = listFirst(clients); + client *receiver = (client*)clientnode->value; + std::unique_locklock)> lock(receiver->lock); + + if (receiver->btype != BLOCKED_ZSET) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + continue; + } + + int where = (receiver->lastcmd && + receiver->lastcmd->proc == bzpopminCommand) + ? ZSET_MIN : ZSET_MAX; + unblockClient(receiver); + genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + zcard--; + + /* Replicate the command. */ + robj *argv[2]; + struct redisCommand *cmd = where == ZSET_MIN ? + cserver.zpopminCommand : + cserver.zpopmaxCommand; + argv[0] = createStringObject(cmd->name,strlen(cmd->name)); + argv[1] = rl->key; + incrRefCount(rl->key); + propagate(cmd,receiver->db->id, + argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + decrRefCount(argv[0]); + decrRefCount(argv[1]); + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a stream key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + stream *s = (stream*)ptrFromObj(o); + + /* We need to provide the new data arrived on the stream + * to all the clients that are waiting for an offset smaller + * than the current top item. */ + if (de) { + list *clients = (list*)dictGetVal(de); + listNode *ln; + listIter li; + listRewind(clients,&li); + + while((ln = listNext(&li))) { + client *receiver = (client*)listNodeValue(ln); + if (receiver->btype != BLOCKED_STREAM) continue; + std::unique_locklock)> lock(receiver->lock); + streamID *gt = (streamID*)dictFetchValue(receiver->bpop.keys, + rl->key); + + /* If we blocked in the context of a consumer + * group, we need to resolve the group and update the + * last ID the client is blocked for: this is needed + * because serving other clients in the same consumer + * group will alter the "last ID" of the consumer + * group, and clients blocked in a consumer group are + * always blocked for the ">" ID: we need to deliver + * only new messages and avoid unblocking the client + * otherwise. */ + streamCG *group = NULL; + if (receiver->bpop.xread_group) { + group = streamLookupCG(s, + szFromObj(receiver->bpop.xread_group)); + /* If the group was not found, send an error + * to the consumer. */ + if (!group) { + addReplyErrorAsync(receiver, + "-NOGROUP the consumer group this client " + "was blocked on no longer exists"); + unblockClient(receiver); + continue; + } else { + *gt = group->last_id; + } + } + + if (streamCompareID(&s->last_id, gt) > 0) { + streamID start = *gt; + streamIncrID(&start); + + /* Lookup the consumer for the group, if any. */ + streamConsumer *consumer = NULL; + int noack = 0; + + if (group) { + consumer = streamLookupConsumer(group, + szFromObj(receiver->bpop.xread_consumer), + 1); + noack = receiver->bpop.xread_group_noack; + } + + /* Emit the two elements sub-array consisting of + * the name of the stream and the data we + * extracted from it. Wrapped in a single-item + * array, since we have just one key. */ + if (receiver->resp == 2) { + addReplyArrayLenAsync(receiver,1); + addReplyArrayLenAsync(receiver,2); + } else { + addReplyMapLenAsync(receiver,1); + } + addReplyBulkAsync(receiver,rl->key); + + streamPropInfo pi = { + rl->key, + receiver->bpop.xread_group + }; + streamReplyWithRange(receiver,s,&start,NULL, + receiver->bpop.xread_count, + 0, group, consumer, noack, &pi); + + /* Note that after we unblock the client, 'gt' + * and other receiver->bpop stuff are no longer + * valid, so we must do the setup above before + * this call. */ + unblockClient(receiver); + } + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * in order to check if we can serve clients blocked by modules using + * RM_BlockClientOnKeys(), when the corresponding key was signaled as ready: + * our goal here is to call the RedisModuleBlockedClient reply() callback to + * see if the key is really able to serve the client, and in that case, + * unblock it. */ +void serveClientsBlockedOnKeyByModule(readyList *rl) { + dictEntry *de; + + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = (list*)dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = (client*)clientnode->value; + + /* Put at the tail, so that at the next call + * we'll not run into it again: clients here may not be + * ready to be served, so they'll remain in the list + * sometimes. We want also be able to skip clients that are + * not blocked for the MODULE type safely. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + + if (receiver->btype != BLOCKED_MODULE) continue; + + /* Note that if *this* client cannot be served by this key, + * it does not mean that another client that is next into the + * list cannot be served as well: they may be blocked by + * different modules with different triggers to consider if a key + * is ready or not. This means we can't exit the loop but need + * to continue after the first failure. */ + if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; + + moduleUnblockClient(receiver); + } + } +} + /* This function should be called by Redis every time a single command, * a MULTI/EXEC block, or a Lua script, terminated its execution after * being called by a client. It handles serving clients blocked in @@ -288,215 +536,32 @@ void handleClientsBlockedOnKeys(void) { * we can safely call signalKeyAsReady() against this key. */ dictDelete(rl->db->ready_keys,rl->key); + /* Even if we are not inside call(), increment the call depth + * in order to make sure that keys are expired against a fixed + * reference time, and not against the wallclock time. This + * way we can lookup an object multiple times (BRPOPLPUSH does + * that) without the risk of it being freed in the second + * lookup, invalidating the first one. + * See https://github.com/antirez/redis/pull/6554. */ + serverTL->fixed_time_expire++; + updateCachedTime(0); + /* Serve clients blocked on list key. */ robj *o = lookupKeyWrite(rl->db,rl->key); - if (o != NULL && o->type == OBJ_LIST) { - dictEntry *de; - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = (list*)dictGetVal(de); - int numclients = listLength(clients); - - while(numclients--) { - listNode *clientnode = listFirst(clients); - client *receiver = (client*)clientnode->value; - std::unique_lock lock(receiver->lock); - - if (receiver->btype != BLOCKED_LIST) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - robj *dstkey = receiver->bpop.target; - int where = (receiver->lastcmd && - receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; - robj *value = listTypePop(o,where); - - if (value) { - /* Protect receiver->bpop.target, that will be - * freed by the next unblockClient() - * call. */ - if (dstkey) incrRefCount(dstkey); - fastlock_lock(&receiver->lock); - unblockClient(receiver); - - if (serveClientBlockedOnList(receiver, - rl->key,dstkey,rl->db,value, - where) == C_ERR) - { - /* If we failed serving the client we need - * to also undo the POP operation. */ - listTypePush(o,value,where); - } - - if (dstkey) decrRefCount(dstkey); - fastlock_unlock(&receiver->lock); - decrRefCount(value); - } else { - break; - } - } - } - - if (listTypeLength(o) == 0) { - dbDelete(rl->db,rl->key); - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); - } - /* We don't call signalModifiedKey() as it was already called - * when an element was pushed on the list. */ - } - - /* Serve clients blocked on sorted set key. */ - else if (o != NULL && o->type == OBJ_ZSET) { - dictEntry *de; - - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = (list*)dictGetVal(de); - int numclients = listLength(clients); - unsigned long zcard = zsetLength(o); - - while(numclients-- && zcard) { - listNode *clientnode = listFirst(clients); - client *receiver = (client*)clientnode->value; - - if (receiver->btype != BLOCKED_ZSET) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - fastlock_lock(&receiver->lock); - int where = (receiver->lastcmd && - receiver->lastcmd->proc == bzpopminCommand) - ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); - genericZpopCommand(receiver,&rl->key,1,where,1,NULL); - zcard--; - - /* Replicate the command. */ - robj *argv[2]; - struct redisCommand *cmd = where == ZSET_MIN ? - cserver.zpopminCommand : - cserver.zpopmaxCommand; - argv[0] = createStringObject(cmd->name,strlen(cmd->name)); - argv[1] = rl->key; - incrRefCount(rl->key); - propagate(cmd,receiver->db->id, - argv,2,PROPAGATE_AOF|PROPAGATE_REPL); - fastlock_unlock(&receiver->lock); - decrRefCount(argv[0]); - decrRefCount(argv[1]); - } - } - } - - /* Serve clients blocked on stream key. */ - else if (o != NULL && o->type == OBJ_STREAM) { - dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); - stream *s = (stream*)ptrFromObj(o); - - /* We need to provide the new data arrived on the stream - * to all the clients that are waiting for an offset smaller - * than the current top item. */ - if (de) { - list *clients = (list*)dictGetVal(de); - listNode *ln; - listIter li; - listRewind(clients,&li); - - while((ln = listNext(&li))) { - client *receiver = (client*)listNodeValue(ln); - if (receiver->btype != BLOCKED_STREAM) continue; - streamID *gt = (streamID*)dictFetchValue(receiver->bpop.keys, - rl->key); - - /* If we blocked in the context of a consumer - * group, we need to resolve the group and update the - * last ID the client is blocked for: this is needed - * because serving other clients in the same consumer - * group will alter the "last ID" of the consumer - * group, and clients blocked in a consumer group are - * always blocked for the ">" ID: we need to deliver - * only new messages and avoid unblocking the client - * otherwise. */ - streamCG *group = NULL; - if (receiver->bpop.xread_group) { - group = streamLookupCG(s, - szFromObj(receiver->bpop.xread_group)); - /* If the group was not found, send an error - * to the consumer. */ - if (!group) { - fastlock_lock(&receiver->lock); - addReplyErrorAsync(receiver, - "-NOGROUP the consumer group this client " - "was blocked on no longer exists"); - unblockClient(receiver); - fastlock_unlock(&receiver->lock); - continue; - } else { - *gt = group->last_id; - } - } - - if (streamCompareID(&s->last_id, gt) > 0) { - streamID start = *gt; - start.seq++; /* Can't overflow, it's an uint64_t */ - - /* Lookup the consumer for the group, if any. */ - streamConsumer *consumer = NULL; - int noack = 0; - - if (group) { - consumer = streamLookupConsumer(group, - szFromObj(receiver->bpop.xread_consumer), - 1); - noack = receiver->bpop.xread_group_noack; - } - - fastlock_lock(&receiver->lock); - - /* Emit the two elements sub-array consisting of - * the name of the stream and the data we - * extracted from it. Wrapped in a single-item - * array, since we have just one key. */ - if (receiver->resp == 2) { - addReplyArrayLenAsync(receiver,1); - addReplyArrayLenAsync(receiver,2); - } else { - addReplyMapLenAsync(receiver,1); - } - addReplyBulkAsync(receiver,rl->key); - - streamPropInfo pi = { - rl->key, - receiver->bpop.xread_group - }; - streamReplyWithRange(receiver,s,&start,NULL, - receiver->bpop.xread_count, - 0, group, consumer, noack, &pi); - - /* Note that after we unblock the client, 'gt' - * and other receiver->bpop stuff are no longer - * valid, so we must do the setup above before - * this call. */ - unblockClient(receiver); - fastlock_unlock(&receiver->lock); - } - } - } + if (o != NULL) { + if (o->type == OBJ_LIST) + serveClientsBlockedOnListKey(o,rl); + else if (o->type == OBJ_ZSET) + serveClientsBlockedOnSortedSetKey(o,rl); + else if (o->type == OBJ_STREAM) + serveClientsBlockedOnStreamKey(o,rl); + /* We want to serve clients blocked on module keys + * regardless of the object type: we don't know what the + * module is trying to accomplish right now. */ + serveClientsBlockedOnKeyByModule(rl); } + serverTL->fixed_time_expire--; /* Free this item. */ decrRefCount(rl->key); @@ -621,7 +686,7 @@ void unblockClientWaitingData(client *c) { * the same key again and again in the list in case of multiple pushes * made by a script or in the context of MULTI/EXEC. * - * The list will be finally processed by handleClientsBlockedOnLists() */ + * The list will be finally processed by handleClientsBlockedOnKeys() */ void signalKeyAsReady(redisDb *db, robj *key) { readyList *rl; diff --git a/src/childinfo.cpp b/src/childinfo.cpp index 16f6aeb53..66ad8b8fd 100644 --- a/src/childinfo.cpp +++ b/src/childinfo.cpp @@ -80,6 +80,8 @@ void receiveChildInfo(void) { g_pserver->stat_rdb_cow_bytes = g_pserver->child_info_data.cow_size; } else if (g_pserver->child_info_data.process_type == CHILD_INFO_TYPE_AOF) { g_pserver->stat_aof_cow_bytes = g_pserver->child_info_data.cow_size; + } else if (g_pserver->child_info_data.process_type == CHILD_INFO_TYPE_MODULE) { + g_pserver->stat_module_cow_bytes = g_pserver->child_info_data.cow_size; } } } diff --git a/src/cluster.cpp b/src/cluster.cpp index f6a6e03dc..945e07946 100644 --- a/src/cluster.cpp +++ b/src/cluster.cpp @@ -49,7 +49,7 @@ clusterNode *myself = NULL; clusterNode *createClusterNode(char *nodename, int flags); int clusterAddNode(clusterNode *node); void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask); +void clusterReadHandler(connection *conn); void clusterSendPing(clusterLink *link, int type); void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); @@ -146,6 +146,7 @@ int clusterLoadConfig(char *filename) { /* Handle the special "vars" line. Don't pretend it is the last * line even if it actually is when generated by Redis. */ if (strcasecmp(argv[0],"vars") == 0) { + if (!(argc % 2)) goto fmterr; for (j = 1; j < argc; j += 2) { if (strcasecmp(argv[j],"currentEpoch") == 0) { g_pserver->cluster->currentEpoch = @@ -489,7 +490,8 @@ void clusterInit(void) { /* Port sanity check II * The other handshake port check is triggered too late to stop * us from trying to use a too-high cluster port number. */ - if (g_pserver->port > (65535-CLUSTER_PORT_INCR)) { + int port = g_pserver->tls_cluster ? g_pserver->tls_port : g_pserver->port; + if (port > (65535-CLUSTER_PORT_INCR)) { serverLog(LL_WARNING, "Redis port number too high. " "Cluster communication port is 10,000 port " "numbers higher than your Redis port. " @@ -497,8 +499,7 @@ void clusterInit(void) { "lower than 55535."); exit(1); } - - if (listenToPort(g_pserver->port+CLUSTER_PORT_INCR, + if (listenToPort(port+CLUSTER_PORT_INCR, g_pserver->cfd,&g_pserver->cfd_count, 0 /*fReusePort*/, 0 /*fFirstListen*/) == C_ERR) { exit(1); @@ -520,8 +521,8 @@ void clusterInit(void) { /* Set myself->port / cport to my listening ports, we'll just need to * discover the IP address via MEET messages. */ - myself->port = g_pserver->port; - myself->cport = g_pserver->port+CLUSTER_PORT_INCR; + myself->port = port; + myself->cport = port+CLUSTER_PORT_INCR; if (g_pserver->cluster_announce_port) myself->port = g_pserver->cluster_announce_port; if (g_pserver->cluster_announce_bus_port) @@ -609,7 +610,7 @@ clusterLink *createClusterLink(clusterNode *node) { link->sndbuf = sdsempty(); link->rcvbuf = sdsempty(); link->node = node; - link->fd = -1; + link->conn = NULL; return link; } @@ -628,23 +629,45 @@ void freeClusterLink(clusterLink *link) { }); return; } - if (link->fd != -1) { - aeDeleteFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, link->fd, AE_READABLE|AE_WRITABLE); + if (link->conn) { + connClose(link->conn); + link->conn = NULL; } sdsfree(link->sndbuf); sdsfree(link->rcvbuf); if (link->node) link->node->link = NULL; - close(link->fd); zfree(link); } +static void clusterConnAcceptHandler(connection *conn) { + clusterLink *link; + + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", connGetLastError(conn)); + connClose(conn); + return; + } + + /* Create a link object we use to handle the connection. + * It gets passed to the readable handler when data is available. + * Initiallly the link->node pointer is set to NULL as we don't know + * which node is, but the right node is references once we know the + * node identity. */ + link = createClusterLink(NULL); + link->conn = conn; + connSetPrivateData(conn, link); + + /* Register read handler */ + connSetReadHandler(conn, clusterReadHandler); +} + #define MAX_CLUSTER_ACCEPTS_PER_CALL 1000 void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { int cport, cfd; int max = MAX_CLUSTER_ACCEPTS_PER_CALL; char cip[NET_IP_STR_LEN]; - clusterLink *link; UNUSED(el); UNUSED(mask); UNUSED(privdata); @@ -661,19 +684,24 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Error accepting cluster node: %s", serverTL->neterr); return; } - anetNonBlock(NULL,cfd); - anetEnableTcpNoDelay(NULL,cfd); + + connection *conn = g_pserver->tls_cluster ? connCreateAcceptedTLS(cfd,1) : connCreateAcceptedSocket(cfd); + connNonBlock(conn); + connEnableTcpNoDelay(conn); /* Use non-blocking I/O for cluster messages. */ - serverLog(LL_VERBOSE,"Accepted cluster node %s:%d", cip, cport); - /* Create a link object we use to handle the connection. - * It gets passed to the readable handler when data is available. - * Initiallly the link->node pointer is set to NULL as we don't know - * which node is, but the right node is references once we know the - * node identity. */ - link = createClusterLink(NULL); - link->fd = cfd; - aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el,cfd,AE_READABLE,clusterReadHandler,link); + serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport); + + /* Accept the connection now. connAccept() may call our handler directly + * or schedule it for later depending on connection implementation. + */ + if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) { + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", + connGetLastError(conn)); + connClose(conn); + return; + } } } @@ -1474,7 +1502,7 @@ void nodeIp2String(char *buf, clusterLink *link, char *announced_ip) { memcpy(buf,announced_ip,NET_IP_STR_LEN); buf[NET_IP_STR_LEN-1] = '\0'; /* We are not sure the input is sane. */ } else { - anetPeerToString(link->fd, buf, NET_IP_STR_LEN, NULL); + connPeerToString(link->conn, buf, NET_IP_STR_LEN, NULL); } } @@ -1782,7 +1810,7 @@ int clusterProcessPacket(clusterLink *link) { { char ip[NET_IP_STR_LEN]; - if (anetSockName(link->fd,ip,sizeof(ip),NULL) != -1 && + if (connSockName(link->conn,ip,sizeof(ip),NULL) != -1 && strcmp(ip,myself->ip)) { memcpy(myself->ip,ip,NET_IP_STR_LEN); @@ -2149,12 +2177,10 @@ void handleLinkIOError(clusterLink *link) { /* Send data. This is handled using a trivial send buffer that gets * consumed by write(). We don't try to optimize this for speed too much * as this is a very low traffic channel. */ -void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - serverAssert(ielFromEventLoop(el) == IDX_EVENT_LOOP_MAIN); - clusterLink *link = (clusterLink*) privdata; +void clusterWriteHandler(connection *conn) { + serverAssert(ielFromEventLoop(serverTL->el) == IDX_EVENT_LOOP_MAIN); + clusterLink *link = (clusterLink*)connGetPrivateData(conn); ssize_t nwritten; - UNUSED(el); - UNUSED(mask); // We're about to release the lock, so the link's sndbuf needs to be owned fully by us // allocate a new one in case anyone tries to write while we're waiting @@ -2162,12 +2188,12 @@ void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { link->sndbuf = sdsempty(); aeReleaseLock(); - nwritten = write(fd, sndbuf, sdslen(sndbuf)); + nwritten = connWrite(conn, link->sndbuf, sdslen(link->sndbuf)); aeAcquireLock(); if (nwritten <= 0) { serverLog(LL_DEBUG,"I/O error writing to node link: %s", - (nwritten == -1) ? strerror(errno) : "short write"); + (nwritten == -1) ? connGetLastError(conn) : "short write"); sdsfree(sndbuf); handleLinkIOError(link); return; @@ -2178,20 +2204,63 @@ void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { sdsfree(link->sndbuf); link->sndbuf = sndbuf; if (sdslen(link->sndbuf) == 0) - aeDeleteFileEvent(el, link->fd, AE_WRITABLE); + connSetWriteHandler(link->conn, NULL); +} + +/* A connect handler that gets called when a connection to another node + * gets established. + */ +void clusterLinkConnectHandler(connection *conn) { + clusterLink *link = (clusterLink*)connGetPrivateData(conn); + clusterNode *node = link->node; + + /* Check if connection succeeded */ + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, "Connection with Node %.40s at %s:%d failed: %s", + node->name, node->ip, node->cport, + connGetLastError(conn)); + freeClusterLink(link); + return; + } + + /* Register a read handler from now on */ + connSetReadHandler(conn, clusterReadHandler); + + /* Queue a PING in the new connection ASAP: this is crucial + * to avoid false positives in failure detection. + * + * If the node is flagged as MEET, we send a MEET message instead + * of a PING one, to force the receiver to add us in its node + * table. */ + mstime_t old_ping_sent = node->ping_sent; + clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? + CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); + if (old_ping_sent) { + /* If there was an active ping before the link was + * disconnected, we want to restore the ping time, otherwise + * replaced by the clusterSendPing() call. */ + node->ping_sent = old_ping_sent; + } + /* We can clear the flag after the first packet is sent. + * If we'll never receive a PONG, we'll never send new packets + * to this node. Instead after the PONG is received and we + * are no longer in meet/handshake status, we want to send + * normal PING packets. */ + node->flags &= ~CLUSTER_NODE_MEET; + + serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", + node->name, node->ip, node->cport); } /* Read data. Try to read the first field of the header first to check the * full length of the packet. When a whole packet is in memory this function * will call the function to process the packet. And so forth. */ -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - char buf[sizeof(clusterMsg)]; +void clusterReadHandler(connection *conn) { + clusterMsg buf[1]; ssize_t nread; clusterMsg *hdr; - clusterLink *link = (clusterLink*) privdata; + clusterLink *link = (clusterLink*)connGetPrivateData(conn); unsigned int readlen, rcvbuflen; - UNUSED(el); - UNUSED(mask); while(1) { /* Read as long as there is data to read. */ rcvbuflen = sdslen(link->rcvbuf); @@ -2219,13 +2288,13 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { if (readlen > sizeof(buf)) readlen = sizeof(buf); } - nread = read(fd,buf,readlen); - if (nread == -1 && errno == EAGAIN) return; /* No more data ready. */ + nread = connRead(conn,buf,readlen); + if (nread == -1 && (connGetState(conn) == CONN_STATE_CONNECTED)) return; /* No more data ready. */ if (nread <= 0) { /* I/O error... */ serverLog(LL_DEBUG,"I/O error reading from node link: %s", - (nread == 0) ? "connection closed" : strerror(errno)); + (nread == 0) ? "connection closed" : connGetLastError(conn)); handleLinkIOError(link); return; } else { @@ -2255,8 +2324,7 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { serverAssert(GlobalLocksAcquired()); if (sdslen(link->sndbuf) == 0 && msglen != 0) - aeCreateRemoteFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el,link->fd,AE_WRITABLE|AE_BARRIER, - clusterWriteHandler,link,false); + connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1); link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); @@ -2322,11 +2390,12 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) { } /* Handle cluster-announce-port as well. */ + int port = g_pserver->tls_cluster ? g_pserver->tls_port : g_pserver->port; int announced_port = g_pserver->cluster_announce_port ? - g_pserver->cluster_announce_port : g_pserver->port; + g_pserver->cluster_announce_port : port; int announced_cport = g_pserver->cluster_announce_bus_port ? g_pserver->cluster_announce_bus_port : - (g_pserver->port + CLUSTER_PORT_INCR); + (port + CLUSTER_PORT_INCR); memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots)); memset(hdr->slaveof,0,CLUSTER_NAMELEN); @@ -2563,7 +2632,8 @@ void clusterBroadcastPong(int target) { * * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { - unsigned char buf[sizeof(clusterMsg)], *payload; + unsigned char *payload; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; uint32_t channel_len, message_len; @@ -2583,7 +2653,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - payload = buf; + payload = (unsigned char*)buf; } else { payload = (unsigned char*)zmalloc(totlen, MALLOC_LOCAL); memcpy(payload,hdr,sizeof(*hdr)); @@ -2600,7 +2670,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { decrRefCount(channel); decrRefCount(message); - if (payload != buf) zfree(payload); + if (payload != (unsigned char*)buf) zfree(payload); } /* Send a FAIL message to all the nodes we are able to contact. @@ -2609,7 +2679,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { * we switch the node state to CLUSTER_NODE_FAIL and ask all the other * nodes to do the same ASAP. */ void clusterSendFail(char *nodename) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAIL); @@ -2621,7 +2691,7 @@ void clusterSendFail(char *nodename) { * slots configuration. The node name, slots bitmap, and configEpoch info * are included. */ void clusterSendUpdate(clusterLink *link, clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; if (link == NULL) return; @@ -2629,7 +2699,7 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { memcpy(hdr->data.update.nodecfg.nodename,node->name,CLUSTER_NAMELEN); hdr->data.update.nodecfg.configEpoch = htonu64(node->configEpoch); memcpy(hdr->data.update.nodecfg.slots,node->slots,sizeof(node->slots)); - clusterSendMessage(link,buf,ntohl(hdr->totlen)); + clusterSendMessage(link,(unsigned char*)buf,ntohl(hdr->totlen)); } /* Send a MODULE message. @@ -2637,7 +2707,8 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, unsigned char *payload, uint32_t len) { - unsigned char buf[sizeof(clusterMsg)], *heapbuf; + unsigned char *heapbuf; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2652,7 +2723,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - heapbuf = buf; + heapbuf = (unsigned char*)buf; } else { heapbuf = (unsigned char*)zmalloc(totlen, MALLOC_LOCAL); memcpy(heapbuf,hdr,sizeof(*hdr)); @@ -2665,7 +2736,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, else clusterBroadcastMessage(heapbuf,totlen); - if (heapbuf != buf) zfree(heapbuf); + if (heapbuf != (unsigned char*)buf) zfree(heapbuf); } /* This function gets a cluster node ID string as target, the same way the nodes @@ -2709,7 +2780,7 @@ void clusterPropagatePublish(robj *channel, robj *message) { * Note that we send the failover request to everybody, master and slave nodes, * but only the masters are supposed to reply to our query. */ void clusterRequestFailoverAuth(void) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2725,7 +2796,7 @@ void clusterRequestFailoverAuth(void) { /* Send a FAILOVER_AUTH_ACK message to the specified node. */ void clusterSendFailoverAuth(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2733,12 +2804,12 @@ void clusterSendFailoverAuth(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Send a MFSTART message to the specified node. */ void clusterSendMFStart(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2746,7 +2817,7 @@ void clusterSendMFStart(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_MFSTART); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Vote for the node asking for our vote if there are the conditions. */ @@ -3429,13 +3500,11 @@ void clusterCron(void) { } if (node->link == NULL) { - int fd; - mstime_t old_ping_sent; - clusterLink *link; - - fd = anetTcpNonBlockBindConnect(serverTL->neterr, node->ip, - node->cport, NET_FIRST_BIND_ADDR); - if (fd == -1) { + clusterLink *link = createClusterLink(node); + link->conn = g_pserver->tls_cluster ? connCreateTLS() : connCreateSocket(); + connSetPrivateData(link->conn, link); + if (connConnect(link->conn, node->ip, node->cport, NET_FIRST_BIND_ADDR, + clusterLinkConnectHandler) == -1) { /* We got a synchronous error from connect before * clusterSendPing() had a chance to be called. * If node->ping_sent is zero, failure detection can't work, @@ -3445,37 +3514,11 @@ void clusterCron(void) { serverLog(LL_DEBUG, "Unable to connect to " "Cluster Node [%s]:%d -> %s", node->ip, node->cport, serverTL->neterr); + + freeClusterLink(link); continue; } - link = createClusterLink(node); - link->fd = fd; node->link = link; - aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el,link->fd,AE_READABLE, - clusterReadHandler,link); - /* Queue a PING in the new connection ASAP: this is crucial - * to avoid false positives in failure detection. - * - * If the node is flagged as MEET, we send a MEET message instead - * of a PING one, to force the receiver to add us in its node - * table. */ - old_ping_sent = node->ping_sent; - clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? - CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); - if (old_ping_sent) { - /* If there was an active ping before the link was - * disconnected, we want to restore the ping time, otherwise - * replaced by the clusterSendPing() call. */ - node->ping_sent = old_ping_sent; - } - /* We can clear the flag after the first packet is sent. - * If we'll never receive a PONG, we'll never send new packets - * to this node. Instead after the PONG is received and we - * are no longer in meet/handshake status, we want to send - * normal PING packets. */ - node->flags &= ~CLUSTER_NODE_MEET; - - serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", - node->name, node->ip, node->cport); } } dictReleaseIterator(di); @@ -4317,12 +4360,9 @@ NULL } } else if (!strcasecmp(szFromObj(c->argv[1]),"nodes") && c->argc == 2) { /* CLUSTER NODES */ - robj *o; - sds ci = clusterGenNodesDescription(0); - - o = createObject(OBJ_STRING,ci); - addReplyBulk(c,o); - decrRefCount(o); + sds nodes = clusterGenNodesDescription(0); + addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); + sdsfree(nodes); } else if (!strcasecmp(szFromObj(c->argv[1]),"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4564,10 +4604,8 @@ NULL "cluster_stats_messages_received:%lld\r\n", tot_msg_received); /* Produce the reply protocol. */ - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } else if (!strcasecmp(szFromObj(c->argv[1]),"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); @@ -4898,7 +4936,7 @@ int verifyDumpPayload(unsigned char *p, size_t len) { * DUMP is actually not used by Redis Cluster but it is the obvious * complement of RESTORE and can be useful for different applications. */ void dumpCommand(client *c) { - robj_roptr o, dumpobj; + robj_roptr o; rio payload; /* Check if the key is here. */ @@ -4911,9 +4949,7 @@ void dumpCommand(client *c) { createDumpPayload(&payload,o,c->argv[1]); /* Transfer to the client */ - dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); + addReplyBulkSds(c,payload.io.buffer.ptr); return; } @@ -4996,7 +5032,7 @@ void restoreCommand(client *c) { if (!absttl) ttl+=mstime(); setExpire(c,c->db,c->argv[1],nullptr,ttl); } - objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock); + objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000); signalModifiedKey(c->db,c->argv[1]); addReply(c,shared.ok); g_pserver->dirty++; @@ -5012,7 +5048,7 @@ void restoreCommand(client *c) { #define MIGRATE_SOCKET_CACHE_TTL 10 /* close cached sockets after 10 sec. */ typedef struct migrateCachedSocket { - int fd; + connection *conn; long last_dbid; time_t last_use_time; } migrateCachedSocket; @@ -5029,7 +5065,7 @@ typedef struct migrateCachedSocket { * should be called so that the connection will be created from scratch * the next time. */ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long timeout) { - int fd; + connection *conn; sds name = sdsempty(); migrateCachedSocket *cs; @@ -5049,34 +5085,26 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti /* Too many items, drop one at random. */ dictEntry *de = dictGetRandomKey(g_pserver->migrate_cached_sockets); cs = (migrateCachedSocket*)dictGetVal(de); - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(g_pserver->migrate_cached_sockets,dictGetKey(de)); } /* Create the socket */ - fd = anetTcpNonBlockConnect(serverTL->neterr,szFromObj(c->argv[1]), - atoi(szFromObj(c->argv[2]))); - if (fd == -1) { - sdsfree(name); - addReplyErrorFormat(c,"Can't connect to target node: %s", - serverTL->neterr); - return NULL; - } - anetEnableTcpNoDelay(serverTL->neterr,fd); - - /* Check if it connects within the specified timeout. */ - if ((aeWait(fd,AE_WRITABLE,timeout) & AE_WRITABLE) == 0) { - sdsfree(name); + conn = g_pserver->tls_cluster ? connCreateTLS() : connCreateSocket(); + if (connBlockingConnect(conn, szFromObj(c->argv[1]), atoi(szFromObj(c->argv[2])), timeout) + != C_OK) { addReplySds(c, sdsnew("-IOERR error or timeout connecting to the client\r\n")); - close(fd); + connClose(conn); + sdsfree(name); return NULL; } + connEnableTcpNoDelay(conn); /* Add to the cache and return it to the caller. */ cs = (migrateCachedSocket*)zmalloc(sizeof(*cs), MALLOC_LOCAL); - cs->fd = fd; + cs->conn = conn; cs->last_dbid = -1; cs->last_use_time = g_pserver->unixtime; dictAdd(g_pserver->migrate_cached_sockets,name,cs); @@ -5097,7 +5125,7 @@ void migrateCloseSocket(robj *host, robj *port) { return; } - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(g_pserver->migrate_cached_sockets,name); sdsfree(name); @@ -5111,7 +5139,7 @@ void migrateCloseTimedoutSockets(void) { migrateCachedSocket *cs = (migrateCachedSocket*)dictGetVal(de); if ((g_pserver->unixtime - cs->last_use_time) > MIGRATE_SOCKET_CACHE_TTL) { - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(g_pserver->migrate_cached_sockets,dictGetKey(de)); } @@ -5303,7 +5331,7 @@ try_again: while ((towrite = sdslen(buf)-pos) > 0) { towrite = (towrite > (64*1024) ? (64*1024) : towrite); - nwritten = syncWrite(cs->fd,buf+pos,towrite,timeout); + nwritten = connSyncWrite(cs->conn,buf+pos,towrite,timeout); if (nwritten != (signed)towrite) { write_error = 1; goto socket_err; @@ -5314,11 +5342,11 @@ try_again: /* Read the AUTH reply if needed. */ - if (password && syncReadLine(cs->fd, buf0, sizeof(buf0), timeout) <= 0) + if (password && connSyncReadLine(cs->conn, buf0, sizeof(buf0), timeout) <= 0) goto socket_err; /* Read the SELECT reply if needed. */ - if (select && syncReadLine(cs->fd, buf1, sizeof(buf1), timeout) <= 0) + if (select && connSyncReadLine(cs->conn, buf1, sizeof(buf1), timeout) <= 0) goto socket_err; /* Allocate the new argument vector that will replace the current command, @@ -5329,7 +5357,7 @@ try_again: /* Read the RESTORE replies. */ for (j = 0; j < num_keys; j++) { - if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { + if (connSyncReadLine(cs->conn, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; } @@ -5519,8 +5547,8 @@ void readwriteCommand(client *c) { * already "down" but it is fragile to rely on the update of the global state, * so we also handle it here. * - * CLUSTER_REDIR_DOWN_STATE if the cluster is down but the user attempts to - * execute a command that addresses one or more keys. */ + * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is + * down but the user attempts to execute a command that addresses one or more keys. */ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *error_code) { clusterNode *n = NULL; robj *firstkey = NULL; @@ -5643,10 +5671,27 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * without redirections or errors in all the cases. */ if (n == NULL) return myself; - /* Cluster is globally down but we got keys? We can't serve the request. */ + /* Cluster is globally down but we got keys? We only serve the request + * if it is a read command and when allow_reads_when_down is enabled. */ if (g_pserver->cluster->state != CLUSTER_OK) { - if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; - return NULL; + if (!g_pserver->cluster_allow_reads_when_down) { + /* The cluster is configured to block commands when the + * cluster is down. */ + if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; + return NULL; + } else if (!(cmd->flags & CMD_READONLY) && !(cmd->proc == evalCommand) + && !(cmd->proc == evalShaCommand)) + { + /* The cluster is configured to allow read only commands + * but this command is neither readonly, nor EVAL or + * EVALSHA. */ + if (error_code) *error_code = CLUSTER_REDIR_DOWN_RO_STATE; + return NULL; + } else { + /* Fall through and allow the command to be executed: + * this happens when g_pserver->cluster_allow_reads_when_down is + * true and the command is a readonly command or EVAL / EVALSHA. */ + } } /* Return the hashslot by reference. */ @@ -5715,6 +5760,8 @@ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_co addReplySds(c,sdsnew("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")); } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down\r\n")); + } else if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { + addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down and only accepts read commands\r\n")); } else if (error_code == CLUSTER_REDIR_DOWN_UNBOUND) { addReplySds(c,sdsnew("-CLUSTERDOWN Hash slot not served\r\n")); } else if (error_code == CLUSTER_REDIR_MOVED || @@ -5750,7 +5797,10 @@ int clusterRedirectBlockedClientIfNeeded(client *c) { dictEntry *de; dictIterator *di; - /* If the cluster is down, unblock the client with the right error. */ + /* If the cluster is down, unblock the client with the right error. + * If the cluster is configured to allow reads on cluster down, we + * still want to emit this error since a write will be required + * to unblock them which may never come. */ if (g_pserver->cluster->state == CLUSTER_FAIL) { clusterRedirectClient(c,NULL,0,CLUSTER_REDIR_DOWN_STATE); return 1; diff --git a/src/cluster.h b/src/cluster.h index ea4f51c78..59b3dc343 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -17,15 +17,10 @@ extern "C" { /* The following defines are amount of time, sometimes expressed as * multiplicators of the node timeout value (when ending with MULT). */ -#define CLUSTER_DEFAULT_NODE_TIMEOUT 15000 -#define CLUSTER_DEFAULT_SLAVE_VALIDITY 10 /* Slave max data age factor. */ -#define CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE 1 -#define CLUSTER_DEFAULT_SLAVE_NO_FAILOVER 0 /* Failover by default. */ #define CLUSTER_FAIL_REPORT_VALIDITY_MULT 2 /* Fail report validity. */ #define CLUSTER_FAIL_UNDO_TIME_MULT 2 /* Undo fail if master is back. */ #define CLUSTER_FAIL_UNDO_TIME_ADD 10 /* Some additional time. */ #define CLUSTER_FAILOVER_DELAY 5 /* Seconds */ -#define CLUSTER_DEFAULT_MIGRATION_BARRIER 1 #define CLUSTER_MF_TIMEOUT 5000 /* Milliseconds to do a manual failover. */ #define CLUSTER_MF_PAUSE_MULT 2 /* Master pause manual failover mult. */ #define CLUSTER_SLAVE_MIGRATION_DELAY 5000 /* Delay for slave migration. */ @@ -38,13 +33,14 @@ extern "C" { #define CLUSTER_REDIR_MOVED 4 /* -MOVED redirection required. */ #define CLUSTER_REDIR_DOWN_STATE 5 /* -CLUSTERDOWN, global state. */ #define CLUSTER_REDIR_DOWN_UNBOUND 6 /* -CLUSTERDOWN, unbound slot. */ +#define CLUSTER_REDIR_DOWN_RO_STATE 7 /* -CLUSTERDOWN, allow reads. */ struct clusterNode; /* clusterLink encapsulates everything needed to talk with a remote node. */ typedef struct clusterLink { mstime_t ctime; /* Link creation time */ - int fd; /* TCP socket file descriptor */ + connection *conn; /* Connection to remote node */ sds sndbuf; /* Packet send buffer */ sds rcvbuf; /* Packet reception buffer */ struct clusterNode *node; /* Node related to this link if any, or NULL */ diff --git a/src/config.cpp b/src/config.cpp index c056b98dc..0fb2ea60a 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -93,6 +93,13 @@ configEnum aof_fsync_enum[] = { {NULL, 0} }; +configEnum repl_diskless_load_enum[] = { + {"disabled", REPL_DISKLESS_LOAD_DISABLED}, + {"on-empty-db", REPL_DISKLESS_LOAD_WHEN_DB_EMPTY}, + {"swapdb", REPL_DISKLESS_LOAD_SWAPDB}, + {NULL, 0} +}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -100,46 +107,108 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {1024*1024*32, 1024*1024*8, 60} /* pubsub */ }; -/* Configuration values that require no special handling to set, get, load or +/* Generic config infrastructure function pointers + * int is_valid_fn(val, err) + * Return 1 when val is valid, and 0 when invalid. + * Optionslly set err to a static error string. + * int update_fn(val, prev, err) + * This function is called only for CONFIG SET command (not at config file parsing) + * It is called after the actual config is applied, + * Return 1 for success, and 0 for failure. + * Optionslly set err to a static error string. + * On failure the config change will be reverted. + */ + +/* Configuration values that require no special handling to set, get, load or * rewrite. */ -typedef struct configYesNo { +typedef struct boolConfigData { + int *config; /* The pointer to the server config this value is stored in */ + const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} boolConfigData; + +typedef struct stringConfigData { + char **config; /* Pointer to the server config this value is stored in. */ + const char *default_value; /* Default value of the config on rewrite. */ + int (*is_valid_fn)(char* val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(char* val, char* prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int convert_empty_to_null; /* Boolean indicating if empty strings should + be stored as a NULL value. */ +} stringConfigData; + +typedef struct enumConfigData { + int *config; /* The pointer to the server config this value is stored in */ + configEnum *enum_value; /* The underlying enum type this data represents */ + const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} enumConfigData; + +typedef enum numericType { + NUMERIC_TYPE_INT, + NUMERIC_TYPE_UINT, + NUMERIC_TYPE_LONG, + NUMERIC_TYPE_ULONG, + NUMERIC_TYPE_LONG_LONG, + NUMERIC_TYPE_ULONG_LONG, + NUMERIC_TYPE_SIZE_T, + NUMERIC_TYPE_SSIZE_T, + NUMERIC_TYPE_OFF_T, + NUMERIC_TYPE_TIME_T, +} numericType; + +typedef struct numericConfigData { + int is_memory; /* Indicates if this value can be loaded as a memory value */ + long long lower_bound; /* The lower bound of this numeric value */ + long long upper_bound; /* The upper bound of this numeric value */ + const long long default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(long long val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(long long val, long long prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ + numericType numeric_type; /* An enum indicating the type of this value */ + union { + int *i; + unsigned int *ui; + long *l; + unsigned long *ul; + long long *ll; + unsigned long long *ull; + size_t *st; + ssize_t *sst; + off_t *ot; + time_t *tt; + } config; /* The pointer to the numeric config this value is stored in */ +} numericConfigData; + +typedef union typeData { + boolConfigData yesno; + stringConfigData string; + enumConfigData enumd; + numericConfigData numeric; +} typeData; + +typedef struct typeInterface { + /* Called on server start, to init the server with default value */ + void (*init)(typeData data); + /* Called on server start, should return 1 on success, 0 on error and should set err */ + int (*load)(typeData data, sds *argc, int argv, const char **err); + /* Called on CONFIG SET, returns 1 on success, 0 on error */ + int (*set)(typeData data, sds value, const char **err); + /* Called on CONFIG GET, required to add output to the client */ + void (*get)(client *c, typeData data); + /* Called on CONFIG REWRITE, required to rewrite the config state */ + void (*rewrite)(typeData data, const char *name, struct rewriteConfigState *state); +} typeInterface; + +typedef struct standardConfig { const char *name; /* The user visible name of this config */ const char *alias; /* An alias that can also be used for this config */ - int *config; /* The pointer to the server config this value is stored in */ const int modifiable; /* Can this value be updated by CONFIG SET? */ - const int default_value; /* The default value of the config on rewrite */ -} configYesNo; + typeInterface interface; /* The function pointers that define the type interface */ + typeData data; /* The type specific data exposed used by the interface */ +} standardConfig; -configYesNo configs_yesno[] = { - /* Non-Modifiable */ - {"rdbchecksum",NULL,&g_pserver->rdb_checksum,0,CONFIG_DEFAULT_RDB_CHECKSUM}, - {"daemonize",NULL,&cserver.daemonize,0,0}, - {"always-show-logo",NULL,&g_pserver->always_show_logo,0,CONFIG_DEFAULT_ALWAYS_SHOW_LOGO}, - /* Modifiable */ - {"protected-mode",NULL,&g_pserver->protected_mode,1,CONFIG_DEFAULT_PROTECTED_MODE}, - {"rdbcompression",NULL,&g_pserver->rdb_compression,1,CONFIG_DEFAULT_RDB_COMPRESSION}, - {"activerehashing",NULL,&g_pserver->activerehashing,1,CONFIG_DEFAULT_ACTIVE_REHASHING}, - {"stop-writes-on-bgsave-error",NULL,&g_pserver->stop_writes_on_bgsave_err,1,CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR}, - {"dynamic-hz",NULL,&g_pserver->dynamic_hz,1,CONFIG_DEFAULT_DYNAMIC_HZ}, - {"lazyfree-lazy-eviction",NULL,&g_pserver->lazyfree_lazy_eviction,1,CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION}, - {"lazyfree-lazy-expire",NULL,&g_pserver->lazyfree_lazy_expire,1,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE}, - {"lazyfree-lazy-server-del",NULL,&g_pserver->lazyfree_lazy_server_del,1,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL}, - {"repl-disable-tcp-nodelay",NULL,&g_pserver->repl_disable_tcp_nodelay,1,CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY}, - {"repl-diskless-sync",NULL,&g_pserver->repl_diskless_sync,1,CONFIG_DEFAULT_REPL_DISKLESS_SYNC}, - {"aof-rewrite-incremental-fsync",NULL,&g_pserver->aof_rewrite_incremental_fsync,1,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC}, - {"no-appendfsync-on-rewrite",NULL,&g_pserver->aof_no_fsync_on_rewrite,1,CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE}, - {"cluster-require-full-coverage",NULL,&g_pserver->cluster_require_full_coverage,CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE}, - {"rdb-save-incremental-fsync",NULL,&g_pserver->rdb_save_incremental_fsync,1,CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC}, - {"aof-load-truncated",NULL,&g_pserver->aof_load_truncated,1,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED}, - {"aof-use-rdb-preamble",NULL,&g_pserver->aof_use_rdb_preamble,1,CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE}, - {"cluster-replica-no-failover","cluster-slave-no-failover",&g_pserver->cluster_slave_no_failover,1,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER}, - {"replica-lazy-flush","slave-lazy-flush",&g_pserver->repl_slave_lazy_flush,1,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH}, - {"replica-serve-stale-data","slave-serve-stale-data",&g_pserver->repl_serve_stale_data,1,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA}, - {"replica-read-only","slave-read-only",&g_pserver->repl_slave_ro,1,CONFIG_DEFAULT_SLAVE_READ_ONLY}, - {"replica-ignore-maxmemory","slave-ignore-maxmemory",&g_pserver->repl_slave_ignore_maxmemory,1,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY}, - {"multi-master",NULL,&g_pserver->enable_multimaster,false,CONFIG_DEFAULT_ENABLE_MULTIMASTER}, - {NULL, NULL, 0, 0} -}; +extern standardConfig configs[]; /*----------------------------------------------------------------------------- * Enum access functions @@ -179,12 +248,19 @@ const char *evictPolicyToString(void) { * Config file parsing *----------------------------------------------------------------------------*/ +int truefalsetoi(char *s) { + if (!strcasecmp(s,"true")) return 1; + else if (!strcasecmp(s,"false")) return 0; + else return -1; +} + int yesnotoi(char *s) { if (!strcasecmp(s,"yes")) return 1; else if (!strcasecmp(s,"no")) return 0; - else return -1; + else return truefalsetoi(s); } + void appendServerSaveParams(time_t seconds, int changes) { g_pserver->saveparams = (saveparam*)zrealloc(g_pserver->saveparams,sizeof(struct saveparam)*(g_pserver->saveparamslen+1), MALLOC_LOCAL); g_pserver->saveparams[g_pserver->saveparamslen].seconds = seconds; @@ -212,6 +288,12 @@ void queueLoadModule(sds path, sds *argv, int argc) { listAddNodeTail(g_pserver->loadmodule_queue,loadmod); } +void initConfigValues() { + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.init(config->data); + } +} + void loadServerConfigFromString(char *config) { const char *err = NULL; int linenum = 0, totlines, i; @@ -246,14 +328,14 @@ void loadServerConfigFromString(char *config) { /* Iterate the configs that are standard */ int match = 0; - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { + for (standardConfig *config = configs; config->name != NULL; config++) { if ((!strcasecmp(argv[0],config->name) || - (config->alias && !strcasecmp(argv[0],config->alias))) && - (argc == 2)) + (config->alias && !strcasecmp(argv[0],config->alias)))) { - if ((*(config->config) = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; + if (!config->interface.load(config->data, argv, argc, &err)) { + goto loaderr; } + match = 1; break; } @@ -265,27 +347,7 @@ void loadServerConfigFromString(char *config) { } /* Execute config directives */ - if (!strcasecmp(argv[0],"timeout") && argc == 2) { - cserver.maxidletime = atoi(argv[1]); - if (cserver.maxidletime < 0) { - err = "Invalid timeout value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-keepalive") && argc == 2) { - cserver.tcpkeepalive = atoi(argv[1]); - if (cserver.tcpkeepalive < 0) { - err = "Invalid tcp-keepalive value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"port") && argc == 2) { - g_pserver->port = atoi(argv[1]); - if (g_pserver->port < 0 || g_pserver->port > 65535) { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-backlog") && argc == 2) { - g_pserver->tcp_backlog = atoi(argv[1]); - if (g_pserver->tcp_backlog < 0) { - err = "Invalid backlog value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"bind") && argc >= 2) { + if (!strcasecmp(argv[0],"bind") && argc >= 2) { int j, addresses = argc-1; if (addresses > CONFIG_BINDADDR_MAX) { @@ -294,8 +356,6 @@ void loadServerConfigFromString(char *config) { for (j = 0; j < addresses; j++) g_pserver->bindaddr[j] = zstrdup(argv[j+1]); g_pserver->bindaddr_count = addresses; - } else if (!strcasecmp(argv[0],"unixsocket") && argc == 2) { - g_pserver->unixsocket = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"unixsocketperm") && argc == 2) { errno = 0; g_pserver->unixsocketperm = (mode_t)strtol(argv[1], NULL, 8); @@ -319,13 +379,6 @@ void loadServerConfigFromString(char *config) { argv[1], strerror(errno)); exit(1); } - } else if (!strcasecmp(argv[0],"loglevel") && argc == 2) { - cserver.verbosity = configEnumGetValue(loglevel_enum,argv[1]); - if (cserver.verbosity == INT_MIN) { - err = "Invalid log level. " - "Must be one of debug, verbose, notice, warning"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"logfile") && argc == 2) { FILE *logfp; @@ -342,157 +395,14 @@ void loadServerConfigFromString(char *config) { } fclose(logfp); } - } else if (!strcasecmp(argv[0],"aclfile") && argc == 2) { - zfree(g_pserver->acl_filename); - g_pserver->acl_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"syslog-enabled") && argc == 2) { - if ((g_pserver->syslog_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"syslog-ident") && argc == 2) { - if (g_pserver->syslog_ident) zfree(g_pserver->syslog_ident); - g_pserver->syslog_ident = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"syslog-facility") && argc == 2) { - g_pserver->syslog_facility = - configEnumGetValue(syslog_facility_enum,argv[1]); - if (g_pserver->syslog_facility == INT_MIN) { - err = "Invalid log facility. Must be one of USER or between LOCAL0-LOCAL7"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"databases") && argc == 2) { - cserver.dbnum = atoi(argv[1]); - if (cserver.dbnum < 1) { - err = "Invalid number of databases"; goto loaderr; - } } else if (!strcasecmp(argv[0],"include") && argc == 2) { loadServerConfig(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) { - g_pserver->maxclients = atoi(argv[1]); - if (g_pserver->maxclients < 1) { - err = "Invalid max clients limit"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { - g_pserver->maxmemory = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) { - g_pserver->maxmemory_policy = - configEnumGetValue(maxmemory_policy_enum,argv[1]); - if (g_pserver->maxmemory_policy == INT_MIN) { - err = "Invalid maxmemory policy"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory-samples") && argc == 2) { - g_pserver->maxmemory_samples = atoi(argv[1]); - if (g_pserver->maxmemory_samples <= 0) { - err = "maxmemory-samples must be 1 or greater"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"proto-max-bulk-len")) && argc == 2) { - g_pserver->proto_max_bulk_len = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { - cserver.client_max_querybuf_len = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"lfu-log-factor") && argc == 2) { - g_pserver->lfu_log_factor = atoi(argv[1]); - if (g_pserver->lfu_log_factor < 0) { - err = "lfu-log-factor must be 0 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lfu-decay-time") && argc == 2) { - g_pserver->lfu_decay_time = atoi(argv[1]); - if (g_pserver->lfu_decay_time < 0) { - err = "lfu-decay-time must be 0 or greater"; - goto loaderr; - } + cserver.client_max_querybuf_len = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"slaveof") || !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; - replicationAddMaster(sdsnew(argv[1]), atoi(argv[2])); - } else if ((!strcasecmp(argv[0],"repl-ping-slave-period") || - !strcasecmp(argv[0],"repl-ping-replica-period")) && - argc == 2) - { - g_pserver->repl_ping_slave_period = atoi(argv[1]); - if (g_pserver->repl_ping_slave_period <= 0) { - err = "repl-ping-replica-period must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-timeout") && argc == 2) { - g_pserver->repl_timeout = atoi(argv[1]); - if (g_pserver->repl_timeout <= 0) { - err = "repl-timeout must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-diskless-sync-delay") && argc==2) { - g_pserver->repl_diskless_sync_delay = atoi(argv[1]); - if (g_pserver->repl_diskless_sync_delay < 0) { - err = "repl-diskless-sync-delay can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { - long long size = memtoll(argv[1],NULL); - if (size <= 0) { - err = "repl-backlog-size must be 1 or greater."; - goto loaderr; - } - resizeReplicationBacklog(size); - } else if (!strcasecmp(argv[0],"repl-backlog-ttl") && argc == 2) { - g_pserver->repl_backlog_time_limit = atoi(argv[1]); - if (g_pserver->repl_backlog_time_limit < 0) { - err = "repl-backlog-ttl can't be negative "; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"masteruser") && argc == 2) { - zfree(cserver.default_masteruser); - cserver.default_masteruser = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { - zfree(cserver.default_masterauth); - cserver.default_masterauth = argv[1][0] ? zstrdup(argv[1]) : NULL; - // Loop through all existing master infos and update them (in case this came after the replicaof config) - updateMasterAuth(); - } else if (!strcasecmp(argv[0],"activedefrag") && argc == 2) { - if ((cserver.active_defrag_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - if (cserver.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - err = "active defrag can't be enabled without proper jemalloc support"; goto loaderr; -#endif - } - } else if (!strcasecmp(argv[0],"hz") && argc == 2) { - g_pserver->config_hz = atoi(argv[1]); - if (g_pserver->config_hz < CONFIG_MIN_HZ) g_pserver->config_hz = CONFIG_MIN_HZ; - if (g_pserver->config_hz > CONFIG_MAX_HZ) g_pserver->config_hz = CONFIG_MAX_HZ; - } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { - int yes; - - if ((yes = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - g_pserver->aof_state = yes ? AOF_ON : AOF_OFF; - } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "appendfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(g_pserver->aof_filename); - g_pserver->aof_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"appendfsync") && argc == 2) { - g_pserver->aof_fsync = configEnumGetValue(aof_fsync_enum,argv[1]); - if (g_pserver->aof_fsync == INT_MIN) { - err = "argument must be 'no', 'always' or 'everysec'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-percentage") && - argc == 2) - { - g_pserver->aof_rewrite_perc = atoi(argv[1]); - if (g_pserver->aof_rewrite_perc < 0) { - err = "Invalid negative percentage for AOF auto rewrite"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") && - argc == 2) - { - g_pserver->aof_rewrite_min_size = memtoll(argv[1],NULL); + replicationAddMaster(argv[1], atoi(argv[2])); } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; @@ -504,81 +414,10 @@ void loadServerConfigFromString(char *config) { sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); - } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) { - zfree(cserver.pidfile); - cserver.pidfile = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "dbfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(g_pserver->rdb_filename); - g_pserver->rdb_filename = zstrdup(argv[1]); - } else if(!strcasecmp(argv[0],"db-s3-object") && argc == 2) { - zfree(g_pserver->rdb_s3bucketpath); - g_pserver->rdb_s3bucketpath = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"active-defrag-threshold-lower") && argc == 2) { - cserver.active_defrag_threshold_lower = atoi(argv[1]); - if (cserver.active_defrag_threshold_lower < 0 || - cserver.active_defrag_threshold_lower > 1000) { - err = "active-defrag-threshold-lower must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-threshold-upper") && argc == 2) { - cserver.active_defrag_threshold_upper = atoi(argv[1]); - if (cserver.active_defrag_threshold_upper < 0 || - cserver.active_defrag_threshold_upper > 1000) { - err = "active-defrag-threshold-upper must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-ignore-bytes") && argc == 2) { - cserver.active_defrag_ignore_bytes = memtoll(argv[1], NULL); - if (cserver.active_defrag_ignore_bytes <= 0) { - err = "active-defrag-ignore-bytes must above 0"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-min") && argc == 2) { - cserver.active_defrag_cycle_min = atoi(argv[1]); - if (cserver.active_defrag_cycle_min < 1 || cserver.active_defrag_cycle_min > 99) { - err = "active-defrag-cycle-min must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-max") && argc == 2) { - cserver.active_defrag_cycle_max = atoi(argv[1]); - if (cserver.active_defrag_cycle_max < 1 || cserver.active_defrag_cycle_max > 99) { - err = "active-defrag-cycle-max must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-max-scan-fields") && argc == 2) { - cserver.active_defrag_max_scan_fields = strtoll(argv[1],NULL,10); - if (cserver.active_defrag_max_scan_fields < 1) { - err = "active-defrag-max-scan-fields must be positive"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) { - g_pserver->hash_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) { - g_pserver->hash_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-bytes") && argc == 2) { - g_pserver->stream_node_max_bytes = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-entries") && argc == 2) { - g_pserver->stream_node_max_entries = atoi(argv[1]); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { /* DEAD OPTION */ - } else if (!strcasecmp(argv[0],"list-max-ziplist-size") && argc == 2) { - g_pserver->list_max_ziplist_size = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"list-compress-depth") && argc == 2) { - g_pserver->list_compress_depth = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) { - g_pserver->set_max_intset_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) { - g_pserver->zset_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) { - g_pserver->zset_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hll-sparse-max-bytes") && argc == 2) { - g_pserver->hll_sparse_max_bytes = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"rename-command") && argc == 3) { struct redisCommand *cmd = lookupCommand(argv[1]); int retval; @@ -603,72 +442,9 @@ void loadServerConfigFromString(char *config) { err = "Target command name already exists"; goto loaderr; } } - } else if (!strcasecmp(argv[0],"cluster-enabled") && argc == 2) { - if ((g_pserver->cluster_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { zfree(g_pserver->cluster_configfile); g_pserver->cluster_configfile = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-ip") && argc == 2) { - zfree(g_pserver->cluster_announce_ip); - g_pserver->cluster_announce_ip = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-port") && argc == 2) { - g_pserver->cluster_announce_port = atoi(argv[1]); - if (g_pserver->cluster_announce_port < 0 || - g_pserver->cluster_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-announce-bus-port") && - argc == 2) - { - g_pserver->cluster_announce_bus_port = atoi(argv[1]); - if (g_pserver->cluster_announce_bus_port < 0 || - g_pserver->cluster_announce_bus_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-node-timeout") && argc == 2) { - g_pserver->cluster_node_timeout = strtoll(argv[1],NULL,10); - if (g_pserver->cluster_node_timeout <= 0) { - err = "cluster node timeout must be 1 or greater"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-migration-barrier") - && argc == 2) - { - g_pserver->cluster_migration_barrier = atoi(argv[1]); - if (g_pserver->cluster_migration_barrier < 0) { - err = "cluster migration barrier must zero or positive"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"cluster-slave-validity-factor") || - !strcasecmp(argv[0],"cluster-replica-validity-factor")) - && argc == 2) - { - g_pserver->cluster_slave_validity_factor = atoi(argv[1]); - if (g_pserver->cluster_slave_validity_factor < 0) { - err = "cluster replica validity factor must be zero or positive"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { - g_pserver->lua_time_limit = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - g_pserver->lua_always_replicate_commands = yesnotoi(argv[1]); - } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && - argc == 2) - { - g_pserver->slowlog_log_slower_than = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"latency-monitor-threshold") && - argc == 2) - { - g_pserver->latency_monitor_threshold = strtoll(argv[1],NULL,10); - if (g_pserver->latency_monitor_threshold < 0) { - err = "The latency threshold can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { - g_pserver->slowlog_max_len = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { @@ -691,38 +467,6 @@ void loadServerConfigFromString(char *config) { cserver.client_obuf_limits[type].hard_limit_bytes = hard; cserver.client_obuf_limits[type].soft_limit_bytes = soft; cserver.client_obuf_limits[type].soft_limit_seconds = soft_seconds; - } else if ((!strcasecmp(argv[0],"slave-priority") || - !strcasecmp(argv[0],"replica-priority")) && argc == 2) - { - g_pserver->slave_priority = atoi(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-ip") || - !strcasecmp(argv[0],"replica-announce-ip")) && argc == 2) - { - zfree(g_pserver->slave_announce_ip); - g_pserver->slave_announce_ip = zstrdup(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-port") || - !strcasecmp(argv[0],"replica-announce-port")) && argc == 2) - { - g_pserver->slave_announce_port = atoi(argv[1]); - if (g_pserver->slave_announce_port < 0 || - g_pserver->slave_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"min-slaves-to-write") || - !strcasecmp(argv[0],"min-replicas-to-write")) && argc == 2) - { - g_pserver->repl_min_slaves_to_write = atoi(argv[1]); - if (g_pserver->repl_min_slaves_to_write < 0) { - err = "Invalid value for min-replicas-to-write."; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"min-slaves-max-lag") || - !strcasecmp(argv[0],"min-replicas-max-lag")) && argc == 2) - { - g_pserver->repl_min_slaves_max_lag = atoi(argv[1]); - if (g_pserver->repl_min_slaves_max_lag < 0) { - err = "Invalid value for min-replicas-max-lag."; goto loaderr; - } } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { int flags = keyspaceEventsStringToFlags(argv[1]); @@ -731,15 +475,6 @@ void loadServerConfigFromString(char *config) { goto loaderr; } g_pserver->notify_keyspace_events = flags; - } else if (!strcasecmp(argv[0],"supervised") && argc == 2) { - cserver.supervised_mode = - configEnumGetValue(supervised_mode_enum,argv[1]); - - if (cserver.supervised_mode == INT_MIN) { - err = "Invalid option for 'supervised'. " - "Allowed values: 'upstart', 'systemd', 'auto', or 'no'"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"user") && argc >= 2) { int argc_err; if (ACLAppendUserForLoading(argv,argc,&argc_err) == C_ERR) { @@ -890,12 +625,6 @@ void loadServerConfig(char *filename, char *options) { if (err || ll < 0) goto badfmt; \ _var = ll; -#define config_set_enum_field(_name,_var,_enumvar) \ - } else if (!strcasecmp(szFromObj(c->argv[2]),_name)) { \ - int enumval = configEnumGetValue(_enumvar,szFromObj(o)); \ - if (enumval == INT_MIN) goto badfmt; \ - _var = enumval; - #define config_set_special_field(_name) \ } else if (!strcasecmp(szFromObj(c->argv[2]),_name)) { @@ -909,18 +638,19 @@ void configSetCommand(client *c) { robj *o; long long ll; int err; + const char *errstr = NULL; serverAssertWithInfo(c,c->argv[2],sdsEncodedObject(c->argv[2])); serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3])); o = c->argv[3]; /* Iterate the configs that are standard */ - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { + for (standardConfig *config = configs; config->name != NULL; config++) { if(config->modifiable && (!strcasecmp(szFromObj(c->argv[2]),config->name) || - (config->alias && !strcasecmp(szFromObj(c->argv[2]),config->alias)))) + (config->alias && !strcasecmp(szFromObj(c->argv[2]),config->alias)))) { - int yn = yesnotoi(szFromObj(o)); - if (yn == -1) goto badfmt; - *(config->config) = yn; + if (!config->interface.set(config->data,szFromObj(o), &errstr)) { + goto badfmt; + } addReply(c,shared.ok); return; } @@ -929,14 +659,7 @@ void configSetCommand(client *c) { if (0) { /* this starts the config_set macros else-if chain. */ /* Special fields that can't be handled with general macros. */ - config_set_special_field("dbfilename") { - if (!pathIsBaseName(szFromObj(o))) { - addReplyError(c, "dbfilename can't be a path, just a filename"); - return; - } - zfree(g_pserver->rdb_filename); - g_pserver->rdb_filename = zstrdup(szFromObj(o)); - } config_set_special_field("requirepass") { + config_set_special_field("requirepass") { if (sdslen(szFromObj(o)) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; /* The old "requirepass" directive just translates to setting * a password to the default user. */ @@ -944,58 +667,6 @@ void configSetCommand(client *c) { sds aclop = sdscatprintf(sdsempty(),">%s",(char*)ptrFromObj(o)); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); - } config_set_special_field("masteruser") { - zfree(cserver.default_masteruser); - cserver.default_masteruser = ((char*)ptrFromObj(o))[0] ? zstrdup(szFromObj(o)) : NULL; - } config_set_special_field("masterauth") { - zfree(cserver.default_masterauth); - cserver.default_masterauth = ((char*)ptrFromObj(o))[0] ? zstrdup(szFromObj(o)) : NULL; - } config_set_special_field("cluster-announce-ip") { - zfree(g_pserver->cluster_announce_ip); - g_pserver->cluster_announce_ip = ((char*)ptrFromObj(o))[0] ? zstrdup(szFromObj(o)) : NULL; - } config_set_special_field("maxclients") { - int orig_value = g_pserver->maxclients; - - if (getLongLongFromObject(o,&ll) == C_ERR || ll < 1) goto badfmt; - - /* Try to check if the OS is capable of supporting so many FDs. */ - g_pserver->maxclients = ll; - serverAssert(FALSE); - if (ll > orig_value) { - adjustOpenFilesLimit(); - if (g_pserver->maxclients != ll) { - addReplyErrorFormat(c,"The operating system is not able to handle the specified number of clients, try with %d", g_pserver->maxclients); - g_pserver->maxclients = orig_value; - return; - } - if ((unsigned int) aeGetSetSize(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el) < - g_pserver->maxclients + CONFIG_FDSET_INCR) - { - for (int iel = 0; iel < cserver.cthreads; ++iel) - { - if (aeResizeSetSize(g_pserver->rgthreadvar[iel].el, - g_pserver->maxclients + CONFIG_FDSET_INCR) == AE_ERR) - { - addReplyError(c,"The event loop API used by Redis is not able to handle the specified number of clients"); - g_pserver->maxclients = orig_value; - return; - } - } - } - } - } config_set_special_field("appendonly") { - int enable = yesnotoi(szFromObj(o)); - - if (enable == -1) goto badfmt; - if (enable == 0 && g_pserver->aof_state != AOF_OFF) { - stopAppendOnly(); - } else if (enable && g_pserver->aof_state == AOF_OFF) { - if (startAppendOnly() == C_ERR) { - addReplyError(c, - "Unable to turn on AOF. Check server logs."); - return; - } - } } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(szFromObj(o),sdslen(szFromObj(o))," ",1,&vlen); @@ -1085,166 +756,18 @@ void configSetCommand(client *c) { if (flags == -1) goto badfmt; g_pserver->notify_keyspace_events = flags; - } config_set_special_field_with_alias("slave-announce-ip", - "replica-announce-ip") - { - zfree(g_pserver->slave_announce_ip); - g_pserver->slave_announce_ip = ((char*)ptrFromObj(o))[0] ? zstrdup(szFromObj(o)) : NULL; - - /* Boolean fields. - * config_set_bool_field(name,var). */ - } config_set_bool_field( - "activedefrag",cserver.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - if (cserver.active_defrag_enabled) { - cserver.active_defrag_enabled = 0; - addReplyError(c, - "-DISABLED Active defragmentation cannot be enabled: it " - "requires a Redis server compiled with a modified Jemalloc " - "like the one shipped by default with the Redis source " - "distribution"); - return; - } -#endif - /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ - } config_set_numerical_field( - "tcp-keepalive",cserver.tcpkeepalive,0,INT_MAX) { - } config_set_numerical_field( - "maxmemory-samples",g_pserver->maxmemory_samples,1,INT_MAX) { - } config_set_numerical_field( - "lfu-log-factor",g_pserver->lfu_log_factor,0,INT_MAX) { - } config_set_numerical_field( - "lfu-decay-time",g_pserver->lfu_decay_time,0,INT_MAX) { - } config_set_numerical_field( - "timeout",cserver.maxidletime,0,INT_MAX) { - } config_set_numerical_field( - "active-defrag-threshold-lower",cserver.active_defrag_threshold_lower,0,1000) { - } config_set_numerical_field( - "active-defrag-threshold-upper",cserver.active_defrag_threshold_upper,0,1000) { - } config_set_memory_field( - "active-defrag-ignore-bytes",cserver.active_defrag_ignore_bytes) { - } config_set_numerical_field( - "active-defrag-cycle-min",cserver.active_defrag_cycle_min,1,99) { - } config_set_numerical_field( - "active-defrag-cycle-max",cserver.active_defrag_cycle_max,1,99) { - } config_set_numerical_field( - "active-defrag-max-scan-fields",cserver.active_defrag_max_scan_fields,1,LONG_MAX) { - } config_set_numerical_field( - "auto-aof-rewrite-percentage",g_pserver->aof_rewrite_perc,0,INT_MAX){ - } config_set_numerical_field( - "hash-max-ziplist-entries",g_pserver->hash_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "hash-max-ziplist-value",g_pserver->hash_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-bytes",g_pserver->stream_node_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-entries",g_pserver->stream_node_max_entries,0,LLONG_MAX) { - } config_set_numerical_field( - "list-max-ziplist-size",g_pserver->list_max_ziplist_size,INT_MIN,INT_MAX) { - } config_set_numerical_field( - "list-compress-depth",g_pserver->list_compress_depth,0,INT_MAX) { - } config_set_numerical_field( - "set-max-intset-entries",g_pserver->set_max_intset_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-entries",g_pserver->zset_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-value",g_pserver->zset_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "hll-sparse-max-bytes",g_pserver->hll_sparse_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "lua-time-limit",g_pserver->lua_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "slowlog-log-slower-than",g_pserver->slowlog_log_slower_than,-1,LLONG_MAX) { - } config_set_numerical_field( - "slowlog-max-len",ll,0,LONG_MAX) { - /* Cast to unsigned. */ - g_pserver->slowlog_max_len = (unsigned long)ll; - } config_set_numerical_field( - "latency-monitor-threshold",g_pserver->latency_monitor_threshold,0,LLONG_MAX){ - } config_set_numerical_field( - "repl-ping-slave-period",g_pserver->repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-ping-replica-period",g_pserver->repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-timeout",g_pserver->repl_timeout,1,INT_MAX) { - } config_set_numerical_field( - "repl-backlog-ttl",g_pserver->repl_backlog_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "repl-diskless-sync-delay",g_pserver->repl_diskless_sync_delay,0,INT_MAX) { - } config_set_numerical_field( - "slave-priority",g_pserver->slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "replica-priority",g_pserver->slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "slave-announce-port",g_pserver->slave_announce_port,0,65535) { - } config_set_numerical_field( - "replica-announce-port",g_pserver->slave_announce_port,0,65535) { - } config_set_numerical_field( - "min-slaves-to-write",g_pserver->repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-to-write",g_pserver->repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-slaves-max-lag",g_pserver->repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-max-lag",g_pserver->repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "cluster-node-timeout",g_pserver->cluster_node_timeout,0,LLONG_MAX) { - } config_set_numerical_field( - "cluster-announce-port",g_pserver->cluster_announce_port,0,65535) { - } config_set_numerical_field( - "cluster-announce-bus-port",g_pserver->cluster_announce_bus_port,0,65535) { - } config_set_numerical_field( - "cluster-migration-barrier",g_pserver->cluster_migration_barrier,0,INT_MAX){ - } config_set_numerical_field( - "cluster-slave-validity-factor",g_pserver->cluster_slave_validity_factor,0,INT_MAX) { - } config_set_numerical_field( - "cluster-replica-validity-factor",g_pserver->cluster_slave_validity_factor,0,INT_MAX) { - } config_set_numerical_field( - "hz",g_pserver->config_hz,0,INT_MAX) { - /* Hz is more an hint from the user, so we accept values out of range - * but cap them to reasonable values. */ - if (g_pserver->config_hz < CONFIG_MIN_HZ) g_pserver->config_hz = CONFIG_MIN_HZ; - if (g_pserver->config_hz > CONFIG_MAX_HZ) g_pserver->config_hz = CONFIG_MAX_HZ; } config_set_numerical_field( "watchdog-period",ll,0,INT_MAX) { if (ll) enableWatchdog(ll); else disableWatchdog(); - /* Memory fields. * config_set_memory_field(name,var) */ - } config_set_memory_field("maxmemory",g_pserver->maxmemory) { - if (g_pserver->maxmemory) { - if (g_pserver->maxmemory < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); - } - freeMemoryIfNeededAndSafe(); - } - } config_set_memory_field( - "proto-max-bulk-len",g_pserver->proto_max_bulk_len) { } config_set_memory_field( "client-query-buffer-limit",cserver.client_max_querybuf_len) { - } config_set_memory_field("repl-backlog-size",ll) { - resizeReplicationBacklog(ll); - } config_set_memory_field("auto-aof-rewrite-min-size",ll) { - g_pserver->aof_rewrite_min_size = ll; - - /* Enumeration fields. - * config_set_enum_field(name,var,enum_var) */ - } config_set_enum_field( - "loglevel",cserver.verbosity,loglevel_enum) { - } config_set_enum_field( - "maxmemory-policy",g_pserver->maxmemory_policy,maxmemory_policy_enum) { - } config_set_enum_field( - "appendfsync",g_pserver->aof_fsync,aof_fsync_enum) { - /* Everyhing else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", @@ -1257,9 +780,16 @@ void configSetCommand(client *c) { return; badfmt: /* Bad format errors */ - addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", - (char*)ptrFromObj(o), - (char*)ptrFromObj(c->argv[2])); + if (errstr) { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s' - %s", + szFromObj(o), + szFromObj(c->argv[2]), + errstr); + } else { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", + szFromObj(o), + szFromObj(c->argv[2])); + } } /*----------------------------------------------------------------------------- @@ -1291,13 +821,6 @@ badfmt: /* Bad format errors */ } \ } while(0); -#define config_get_enum_field(_name,_var,_enumvar) do { \ - if (stringmatch(pattern,_name,1)) { \ - addReplyBulkCString(c,_name); \ - addReplyBulkCString(c,configEnumGetNameOrUnknown(_enumvar,_var)); \ - matches++; \ - } \ -} while(0); void configGetCommand(client *c) { robj *o = c->argv[2]; @@ -1307,122 +830,29 @@ void configGetCommand(client *c) { int matches = 0; serverAssertWithInfo(c,o,sdsEncodedObject(o)); - /* String values */ - config_get_string_field("dbfilename",g_pserver->rdb_filename); - config_get_string_field("masteruser",cserver.default_masteruser); - config_get_string_field("masterauth",cserver.default_masterauth); - config_get_string_field("cluster-announce-ip",g_pserver->cluster_announce_ip); - config_get_string_field("unixsocket",g_pserver->unixsocket); - config_get_string_field("logfile",g_pserver->logfile); - config_get_string_field("aclfile",g_pserver->acl_filename); - config_get_string_field("pidfile",cserver.pidfile); - config_get_string_field("slave-announce-ip",g_pserver->slave_announce_ip); - config_get_string_field("replica-announce-ip",g_pserver->slave_announce_ip); - config_get_string_field("version-override",KEYDB_SET_VERSION); - - /* Numerical values */ - config_get_numerical_field("maxmemory",g_pserver->maxmemory); - config_get_numerical_field("proto-max-bulk-len",g_pserver->proto_max_bulk_len); - config_get_numerical_field("client-query-buffer-limit",cserver.client_max_querybuf_len); - config_get_numerical_field("maxmemory-samples",g_pserver->maxmemory_samples); - config_get_numerical_field("lfu-log-factor",g_pserver->lfu_log_factor); - config_get_numerical_field("lfu-decay-time",g_pserver->lfu_decay_time); - config_get_numerical_field("timeout",cserver.maxidletime); - config_get_numerical_field("active-defrag-threshold-lower",cserver.active_defrag_threshold_lower); - config_get_numerical_field("active-defrag-threshold-upper",cserver.active_defrag_threshold_upper); - config_get_numerical_field("active-defrag-ignore-bytes",cserver.active_defrag_ignore_bytes); - config_get_numerical_field("active-defrag-cycle-min",cserver.active_defrag_cycle_min); - config_get_numerical_field("active-defrag-cycle-max",cserver.active_defrag_cycle_max); - config_get_numerical_field("active-defrag-max-scan-fields",cserver.active_defrag_max_scan_fields); - config_get_numerical_field("auto-aof-rewrite-percentage", - g_pserver->aof_rewrite_perc); - config_get_numerical_field("auto-aof-rewrite-min-size", - g_pserver->aof_rewrite_min_size); - config_get_numerical_field("hash-max-ziplist-entries", - g_pserver->hash_max_ziplist_entries); - config_get_numerical_field("hash-max-ziplist-value", - g_pserver->hash_max_ziplist_value); - config_get_numerical_field("stream-node-max-bytes", - g_pserver->stream_node_max_bytes); - config_get_numerical_field("stream-node-max-entries", - g_pserver->stream_node_max_entries); - config_get_numerical_field("list-max-ziplist-size", - g_pserver->list_max_ziplist_size); - config_get_numerical_field("list-compress-depth", - g_pserver->list_compress_depth); - config_get_numerical_field("set-max-intset-entries", - g_pserver->set_max_intset_entries); - config_get_numerical_field("zset-max-ziplist-entries", - g_pserver->zset_max_ziplist_entries); - config_get_numerical_field("zset-max-ziplist-value", - g_pserver->zset_max_ziplist_value); - config_get_numerical_field("hll-sparse-max-bytes", - g_pserver->hll_sparse_max_bytes); - config_get_numerical_field("lua-time-limit",g_pserver->lua_time_limit); - config_get_numerical_field("slowlog-log-slower-than", - g_pserver->slowlog_log_slower_than); - config_get_numerical_field("latency-monitor-threshold", - g_pserver->latency_monitor_threshold); - config_get_numerical_field("slowlog-max-len", - g_pserver->slowlog_max_len); - config_get_numerical_field("port",g_pserver->port); - config_get_numerical_field("cluster-announce-port",g_pserver->cluster_announce_port); - config_get_numerical_field("cluster-announce-bus-port",g_pserver->cluster_announce_bus_port); - config_get_numerical_field("tcp-backlog",g_pserver->tcp_backlog); - config_get_numerical_field("databases",cserver.dbnum); - config_get_numerical_field("repl-ping-slave-period",g_pserver->repl_ping_slave_period); - config_get_numerical_field("repl-ping-replica-period",g_pserver->repl_ping_slave_period); - config_get_numerical_field("repl-timeout",g_pserver->repl_timeout); - config_get_numerical_field("repl-backlog-size",g_pserver->repl_backlog_size); - config_get_numerical_field("repl-backlog-ttl",g_pserver->repl_backlog_time_limit); - config_get_numerical_field("maxclients",g_pserver->maxclients); - config_get_numerical_field("watchdog-period",g_pserver->watchdog_period); - config_get_numerical_field("slave-priority",g_pserver->slave_priority); - config_get_numerical_field("replica-priority",g_pserver->slave_priority); - config_get_numerical_field("slave-announce-port",g_pserver->slave_announce_port); - config_get_numerical_field("replica-announce-port",g_pserver->slave_announce_port); - config_get_numerical_field("min-slaves-to-write",g_pserver->repl_min_slaves_to_write); - config_get_numerical_field("min-replicas-to-write",g_pserver->repl_min_slaves_to_write); - config_get_numerical_field("min-slaves-max-lag",g_pserver->repl_min_slaves_max_lag); - config_get_numerical_field("min-replicas-max-lag",g_pserver->repl_min_slaves_max_lag); - config_get_numerical_field("hz",g_pserver->config_hz); - config_get_numerical_field("cluster-node-timeout",g_pserver->cluster_node_timeout); - config_get_numerical_field("cluster-migration-barrier",g_pserver->cluster_migration_barrier); - config_get_numerical_field("cluster-slave-validity-factor",g_pserver->cluster_slave_validity_factor); - config_get_numerical_field("cluster-replica-validity-factor",g_pserver->cluster_slave_validity_factor); - config_get_numerical_field("repl-diskless-sync-delay",g_pserver->repl_diskless_sync_delay); - config_get_numerical_field("tcp-keepalive",cserver.tcpkeepalive); - - /* Bool (yes/no) values */ /* Iterate the configs that are standard */ - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { - config_get_bool_field(config->name, *(config->config)); - if (config->alias) { - config_get_bool_field(config->alias, *(config->config)); + for (standardConfig *config = configs; config->name != NULL; config++) { + if (stringmatch(pattern,config->name,1)) { + addReplyBulkCString(c,config->name); + config->interface.get(c,config->data); + matches++; + } + if (config->alias && stringmatch(pattern,config->alias,1)) { + addReplyBulkCString(c,config->alias); + config->interface.get(c,config->data); + matches++; } } - config_get_bool_field("activedefrag", cserver.active_defrag_enabled); + /* String values */ + config_get_string_field("logfile",g_pserver->logfile); - /* Enum values */ - config_get_enum_field("maxmemory-policy", - g_pserver->maxmemory_policy,maxmemory_policy_enum); - config_get_enum_field("loglevel", - cserver.verbosity,loglevel_enum); - config_get_enum_field("supervised", - cserver.supervised_mode,supervised_mode_enum); - config_get_enum_field("appendfsync", - g_pserver->aof_fsync,aof_fsync_enum); - config_get_enum_field("syslog-facility", - g_pserver->syslog_facility,syslog_facility_enum); + /* Numerical values */ + config_get_numerical_field("client-query-buffer-limit",cserver.client_max_querybuf_len); + config_get_numerical_field("watchdog-period",g_pserver->watchdog_period); /* Everything we can't handle with macros follows. */ - if (stringmatch(pattern,"appendonly",1)) { - addReplyBulkCString(c,"appendonly"); - addReplyBulkCString(c,g_pserver->aof_state == AOF_OFF ? "no" : "yes"); - matches++; - } if (stringmatch(pattern,"dir",1)) { char buf[1024]; @@ -1502,12 +932,10 @@ void configGetCommand(client *c) { matches++; } if (stringmatch(pattern,"notify-keyspace-events",1)) { - robj *flagsobj = createObject(OBJ_STRING, - keyspaceEventsFlagsToString(g_pserver->notify_keyspace_events)); + sds flags = keyspaceEventsFlagsToString(g_pserver->notify_keyspace_events); addReplyBulkCString(c,"notify-keyspace-events"); - addReplyBulk(c,flagsobj); - decrRefCount(flagsobj); + addReplyBulkSds(c,flags); matches++; } if (stringmatch(pattern,"bind",1)) { @@ -1528,6 +956,7 @@ void configGetCommand(client *c) { } matches++; } + setDeferredMapLen(c,replylen,matches); } @@ -1817,18 +1246,6 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, const char *optio rewriteConfigRewriteLine(state,option,line,force); } -/* Rewrite the syslog-facility option. */ -void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { - int value = g_pserver->syslog_facility; - int force = value != LOG_LOCAL0; - const char *name = NULL, *option = "syslog-facility"; - sds line; - - name = configEnumGetNameOrUnknown(syslog_facility_enum,value); - line = sdscatprintf(sdsempty(),"%s %s",option,name); - rewriteConfigRewriteLine(state,option,line,force); -} - /* Rewrite the save option. */ void rewriteConfigSaveOption(struct rewriteConfigState *state) { int j; @@ -2138,89 +1555,22 @@ int rewriteConfig(char *path) { * the rewrite state. */ /* Iterate the configs that are standard */ - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { - rewriteConfigYesNoOption(state,config->name,*(config->config),config->default_value); + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.rewrite(config->data, config->name, state); } - rewriteConfigStringOption(state,"pidfile",cserver.pidfile,CONFIG_DEFAULT_PID_FILE); - rewriteConfigNumericalOption(state,"port",g_pserver->port,CONFIG_DEFAULT_SERVER_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-port",g_pserver->cluster_announce_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-bus-port",g_pserver->cluster_announce_bus_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT); - rewriteConfigNumericalOption(state,"tcp-backlog",g_pserver->tcp_backlog,CONFIG_DEFAULT_TCP_BACKLOG); rewriteConfigBindOption(state); - rewriteConfigStringOption(state,"unixsocket",g_pserver->unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",g_pserver->unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); - rewriteConfigNumericalOption(state,"timeout",cserver.maxidletime,CONFIG_DEFAULT_CLIENT_TIMEOUT); - rewriteConfigNumericalOption(state,"tcp-keepalive",cserver.tcpkeepalive,CONFIG_DEFAULT_TCP_KEEPALIVE); - rewriteConfigNumericalOption(state,"replica-announce-port",g_pserver->slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); - rewriteConfigEnumOption(state,"loglevel",cserver.verbosity,loglevel_enum,CONFIG_DEFAULT_VERBOSITY); rewriteConfigStringOption(state,"logfile",g_pserver->logfile,CONFIG_DEFAULT_LOGFILE); - rewriteConfigStringOption(state,"aclfile",g_pserver->acl_filename,CONFIG_DEFAULT_ACL_FILENAME); - rewriteConfigYesNoOption(state,"syslog-enabled",g_pserver->syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED); - rewriteConfigStringOption(state,"syslog-ident",g_pserver->syslog_ident,CONFIG_DEFAULT_SYSLOG_IDENT); - rewriteConfigSyslogfacilityOption(state); rewriteConfigSaveOption(state); rewriteConfigUserOption(state); - rewriteConfigNumericalOption(state,"databases",cserver.dbnum,CONFIG_DEFAULT_DBNUM); - rewriteConfigStringOption(state,"dbfilename",g_pserver->rdb_filename,CONFIG_DEFAULT_RDB_FILENAME); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state,"replicaof"); - rewriteConfigStringOption(state,"replica-announce-ip",g_pserver->slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); - rewriteConfigStringOption(state,"masteruser",cserver.default_masteruser,NULL); - rewriteConfigStringOption(state,"masterauth",cserver.default_masterauth,NULL); - rewriteConfigStringOption(state,"cluster-announce-ip",g_pserver->cluster_announce_ip,NULL); - rewriteConfigNumericalOption(state,"repl-ping-replica-period",g_pserver->repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); - rewriteConfigNumericalOption(state,"repl-timeout",g_pserver->repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); - rewriteConfigBytesOption(state,"repl-backlog-size",g_pserver->repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); - rewriteConfigBytesOption(state,"repl-backlog-ttl",g_pserver->repl_backlog_time_limit,CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT); - rewriteConfigNumericalOption(state,"repl-diskless-sync-delay",g_pserver->repl_diskless_sync_delay,CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY); - rewriteConfigNumericalOption(state,"replica-priority",g_pserver->slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); - rewriteConfigNumericalOption(state,"min-replicas-to-write",g_pserver->repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); - rewriteConfigNumericalOption(state,"min-replicas-max-lag",g_pserver->repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); rewriteConfigRequirepassOption(state,"requirepass"); - rewriteConfigNumericalOption(state,"maxclients",g_pserver->maxclients,CONFIG_DEFAULT_MAX_CLIENTS); - rewriteConfigBytesOption(state,"maxmemory",g_pserver->maxmemory,CONFIG_DEFAULT_MAXMEMORY); - rewriteConfigBytesOption(state,"proto-max-bulk-len",g_pserver->proto_max_bulk_len,CONFIG_DEFAULT_PROTO_MAX_BULK_LEN); rewriteConfigBytesOption(state,"client-query-buffer-limit",cserver.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); - rewriteConfigEnumOption(state,"maxmemory-policy",g_pserver->maxmemory_policy,maxmemory_policy_enum,CONFIG_DEFAULT_MAXMEMORY_POLICY); - rewriteConfigNumericalOption(state,"maxmemory-samples",g_pserver->maxmemory_samples,CONFIG_DEFAULT_MAXMEMORY_SAMPLES); - rewriteConfigNumericalOption(state,"lfu-log-factor",g_pserver->lfu_log_factor,CONFIG_DEFAULT_LFU_LOG_FACTOR); - rewriteConfigNumericalOption(state,"lfu-decay-time",g_pserver->lfu_decay_time,CONFIG_DEFAULT_LFU_DECAY_TIME); - rewriteConfigNumericalOption(state,"active-defrag-threshold-lower",cserver.active_defrag_threshold_lower,CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER); - rewriteConfigNumericalOption(state,"active-defrag-threshold-upper",cserver.active_defrag_threshold_upper,CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER); - rewriteConfigBytesOption(state,"active-defrag-ignore-bytes",cserver.active_defrag_ignore_bytes,CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES); - rewriteConfigNumericalOption(state,"active-defrag-cycle-min",cserver.active_defrag_cycle_min,CONFIG_DEFAULT_DEFRAG_CYCLE_MIN); - rewriteConfigNumericalOption(state,"active-defrag-cycle-max",cserver.active_defrag_cycle_max,CONFIG_DEFAULT_DEFRAG_CYCLE_MAX); - rewriteConfigNumericalOption(state,"active-defrag-max-scan-fields",cserver.active_defrag_max_scan_fields,CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS); - rewriteConfigYesNoOption(state,"appendonly",g_pserver->aof_state != AOF_OFF,0); - rewriteConfigStringOption(state,"appendfilename",g_pserver->aof_filename,CONFIG_DEFAULT_AOF_FILENAME); - rewriteConfigEnumOption(state,"appendfsync",g_pserver->aof_fsync,aof_fsync_enum,CONFIG_DEFAULT_AOF_FSYNC); - rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",g_pserver->aof_rewrite_perc,AOF_REWRITE_PERC); - rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",g_pserver->aof_rewrite_min_size,AOF_REWRITE_MIN_SIZE); - rewriteConfigNumericalOption(state,"lua-time-limit",g_pserver->lua_time_limit,LUA_SCRIPT_TIME_LIMIT); - rewriteConfigYesNoOption(state,"cluster-enabled",g_pserver->cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",g_pserver->cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); - rewriteConfigNumericalOption(state,"cluster-node-timeout",g_pserver->cluster_node_timeout,CLUSTER_DEFAULT_NODE_TIMEOUT); - rewriteConfigNumericalOption(state,"cluster-migration-barrier",g_pserver->cluster_migration_barrier,CLUSTER_DEFAULT_MIGRATION_BARRIER); - rewriteConfigNumericalOption(state,"cluster-replica-validity-factor",g_pserver->cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); - rewriteConfigNumericalOption(state,"slowlog-log-slower-than",g_pserver->slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); - rewriteConfigNumericalOption(state,"latency-monitor-threshold",g_pserver->latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); - rewriteConfigNumericalOption(state,"slowlog-max-len",g_pserver->slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); rewriteConfigNotifykeyspaceeventsOption(state); - rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",g_pserver->hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"hash-max-ziplist-value",g_pserver->hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"stream-node-max-bytes",g_pserver->stream_node_max_bytes,OBJ_STREAM_NODE_MAX_BYTES); - rewriteConfigNumericalOption(state,"stream-node-max-entries",g_pserver->stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); - rewriteConfigNumericalOption(state,"list-max-ziplist-size",g_pserver->list_max_ziplist_size,OBJ_LIST_MAX_ZIPLIST_SIZE); - rewriteConfigNumericalOption(state,"list-compress-depth",g_pserver->list_compress_depth,OBJ_LIST_COMPRESS_DEPTH); - rewriteConfigNumericalOption(state,"set-max-intset-entries",g_pserver->set_max_intset_entries,OBJ_SET_MAX_INTSET_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",g_pserver->zset_max_ziplist_entries,OBJ_ZSET_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-value",g_pserver->zset_max_ziplist_value,OBJ_ZSET_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",g_pserver->hll_sparse_max_bytes,CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES); - rewriteConfigYesNoOption(state,"activedefrag",cserver.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); rewriteConfigClientoutputbufferlimitOption(state); - rewriteConfigNumericalOption(state,"hz",g_pserver->config_hz,CONFIG_DEFAULT_HZ); - rewriteConfigEnumOption(state,"supervised",cserver.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); rewriteConfigYesNoOption(state,"active-replica",g_pserver->fActiveReplica,CONFIG_DEFAULT_ACTIVE_REPLICA); rewriteConfigStringOption(state, "version-override",KEYDB_SET_VERSION,KEYDB_REAL_VERSION); @@ -2242,6 +1592,763 @@ int rewriteConfig(char *path) { return retval; } +/*----------------------------------------------------------------------------- + * Configs that fit one of the major types and require no special handling + *----------------------------------------------------------------------------*/ +#define LOADBUF_SIZE 256 +static char loadbuf[LOADBUF_SIZE]; + +#define MODIFIABLE_CONFIG 1 +#define IMMUTABLE_CONFIG 0 + +#define embedCommonConfig(config_name, config_alias, is_modifiable) \ + config_name, config_alias, is_modifiable, + +#define embedConfigInterface(initfn, loadfn, setfn, getfn, rewritefn) { \ + initfn, loadfn, setfn, getfn, rewritefn, \ +}, + +/* What follows is the generic config types that are supported. To add a new + * config with one of these types, add it to the standardConfig table with + * the creation macro for each type. + * + * Each type contains the following: + * * A function defining how to load this type on startup. + * * A function defining how to update this type on CONFIG SET. + * * A function defining how to serialize this type on CONFIG SET. + * * A function defining how to rewrite this type on CONFIG REWRITE. + * * A Macro defining how to create this type. + */ + +/* Bool Configs */ +static void boolConfigInit(typeData data) { + *data.yesno.config = data.yesno.default_value; +} + +static int boolConfigLoad(typeData data, sds *argv, int argc, const char **err) { + int yn; + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + if ((yn = yesnotoi(argv[1])) == -1) { + if ((yn = truefalsetoi(argv[1])) == -1) + *err = "argument must be 'yes' or 'no'"; + return 0; + } + if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) + return 0; + *data.yesno.config = yn; + return 1; +} + +static int boolConfigSet(typeData data, sds value, const char **err) { + int yn = yesnotoi(value); + if (yn == -1) return 0; + if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) + return 0; + int prev = *(data.yesno.config); + *(data.yesno.config) = yn; + if (data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { + *(data.yesno.config) = prev; + return 0; + } + return 1; +} + +static void boolConfigGet(client *c, typeData data) { + addReplyBulkCString(c, *data.yesno.config ? "yes" : "no"); +} + +static void boolConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); +} + +#define createBoolConfig(name, alias, modifiable, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + {boolConfigInit, boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite}, \ + { { /* .data.yesno */ \ + &(config_addr), \ + (default), \ + (is_valid), \ + (update), \ + } } \ +} + +/* String Configs */ +static void stringConfigInit(typeData data) { + if (data.string.convert_empty_to_null) { + *data.string.config = data.string.default_value ? zstrdup(data.string.default_value) : NULL; + } else { + *data.string.config = zstrdup(data.string.default_value); + } +} + +static int stringConfigLoad(typeData data, sds *argv, int argc, const char **err) { + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + if (data.string.is_valid_fn && !data.string.is_valid_fn(argv[1], err)) + return 0; + zfree(*data.string.config); + if (data.string.convert_empty_to_null) { + *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL; + } else { + *data.string.config = zstrdup(argv[1]); + } + return 1; +} + +static int stringConfigSet(typeData data, sds value, const char **err) { + if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) + return 0; + char *prev = *data.string.config; + if (data.string.convert_empty_to_null) { + *data.string.config = value[0] ? zstrdup(value) : NULL; + } else { + *data.string.config = zstrdup(value); + } + if (data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { + zfree(*data.string.config); + *data.string.config = prev; + return 0; + } + zfree(prev); + return 1; +} + +static void stringConfigGet(client *c, typeData data) { + addReplyBulkCString(c, *data.string.config ? *data.string.config : ""); +} + +static void stringConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value); +} + +#define ALLOW_EMPTY_STRING 0 +#define EMPTY_STRING_IS_NULL 1 + +#define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(stringConfigInit, stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \ + { .string = { \ + &(config_addr), \ + (default), \ + (is_valid), \ + (update), \ + (empty_to_null), \ + } } \ +} + +/* Enum configs */ +static void configEnumInit(typeData data) { + *data.enumd.config = data.enumd.default_value; +} + +static int configEnumLoad(typeData data, sds *argv, int argc, const char **err) { + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + + int enumval = configEnumGetValue(data.enumd.enum_value, argv[1]); + if (enumval == INT_MIN) { + sds enumerr = sdsnew("argument must be one of the following: "); + configEnum *enumNode = data.enumd.enum_value; + while(enumNode->name != NULL) { + enumerr = sdscatlen(enumerr, enumNode->name, strlen(enumNode->name)); + enumerr = sdscatlen(enumerr, ", ", 2); + enumNode++; + } + + enumerr[sdslen(enumerr) - 2] = '\0'; + + /* Make sure we don't overrun the fixed buffer */ + enumerr[LOADBUF_SIZE - 1] = '\0'; + strncpy(loadbuf, enumerr, LOADBUF_SIZE); + + sdsfree(enumerr); + *err = loadbuf; + return 0; + } + if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) + return 0; + *(data.enumd.config) = enumval; + return 1; +} + +static int configEnumSet(typeData data, sds value, const char **err) { + int enumval = configEnumGetValue(data.enumd.enum_value, value); + if (enumval == INT_MIN) return 0; + if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) + return 0; + int prev = *(data.enumd.config); + *(data.enumd.config) = enumval; + if (data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { + *(data.enumd.config) = prev; + return 0; + } + return 1; +} + +static void configEnumGet(client *c, typeData data) { + addReplyBulkCString(c, configEnumGetNameOrUnknown(data.enumd.enum_value,*data.enumd.config)); +} + +static void configEnumRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); +} + +#define createEnumConfig(name, alias, modifiable, enum, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(configEnumInit, configEnumLoad, configEnumSet, configEnumGet, configEnumRewrite) \ + { .enumd = { \ + &(config_addr), \ + (enum), \ + (default), \ + (is_valid), \ + (update), \ + } } \ +} + +/* Gets a 'long long val' and sets it into the union, using a macro to get + * compile time type check. */ +#define SET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + *(data.numeric.config.i) = (int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + *(data.numeric.config.ui) = (unsigned int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + *(data.numeric.config.l) = (long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + *(data.numeric.config.ul) = (unsigned long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + *(data.numeric.config.ll) = (long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + *(data.numeric.config.ull) = (unsigned long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + *(data.numeric.config.st) = (size_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + *(data.numeric.config.sst) = (ssize_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + *(data.numeric.config.ot) = (off_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + *(data.numeric.config.tt) = (time_t) val; \ + } + +/* Gets a 'long long val' and sets it with the value from the union, using a + * macro to get compile time type check. */ +#define GET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + val = *(data.numeric.config.i); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + val = *(data.numeric.config.ui); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + val = *(data.numeric.config.l); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + val = *(data.numeric.config.ul); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + val = *(data.numeric.config.ll); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + val = *(data.numeric.config.ull); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + val = *(data.numeric.config.st); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + val = *(data.numeric.config.sst); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + val = *(data.numeric.config.ot); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + val = *(data.numeric.config.tt); \ + } + +/* Numeric configs */ +static void numericConfigInit(typeData data) { + SET_NUMERIC_TYPE(data.numeric.default_value) +} + +static int numericBoundaryCheck(typeData data, long long ll, const char **err) { + if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || + data.numeric.numeric_type == NUMERIC_TYPE_UINT || + data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + /* Boundary check for unsigned types */ + unsigned long long ull = ll; + unsigned long long upper_bound = data.numeric.upper_bound; + unsigned long long lower_bound = data.numeric.lower_bound; + if (ull > upper_bound || ull < lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %llu and %llu inclusive", + lower_bound, + upper_bound); + *err = loadbuf; + return 0; + } + } else { + /* Boundary check for signed types */ + if (ll > data.numeric.upper_bound || ll < data.numeric.lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, + data.numeric.upper_bound); + *err = loadbuf; + return 0; + } + } + return 1; +} + +static int numericConfigLoad(typeData data, sds *argv, int argc, const char **err) { + long long ll; + + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + + if (data.numeric.is_memory) { + int memerr; + ll = memtoll(argv[1], &memerr); + if (memerr || ll < 0) { + *err = "argument must be a memory value"; + return 0; + } + } else { + if (!string2ll(argv[1], sdslen(argv[1]),&ll)) { + *err = "argument couldn't be parsed into an integer" ; + return 0; + } + } + + if (!numericBoundaryCheck(data, ll, err)) + return 0; + + if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) + return 0; + + SET_NUMERIC_TYPE(ll) + + return 1; +} + +static int numericConfigSet(typeData data, sds value, const char **err) { + long long ll, prev = 0; + if (data.numeric.is_memory) { + int memerr; + ll = memtoll(value, &memerr); + if (memerr || ll < 0) return 0; + } else { + if (!string2ll(value, sdslen(value),&ll)) return 0; + } + + if (!numericBoundaryCheck(data, ll, err)) + return 0; + + if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) + return 0; + + GET_NUMERIC_TYPE(prev) + SET_NUMERIC_TYPE(ll) + + if (data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { + SET_NUMERIC_TYPE(prev) + return 0; + } + return 1; +} + +static void numericConfigGet(client *c, typeData data) { + char buf[128]; + long long value = 0; + + GET_NUMERIC_TYPE(value) + + ll2string(buf, sizeof(buf), value); + addReplyBulkCString(c, buf); +} + +static void numericConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + long long value = 0; + + GET_NUMERIC_TYPE(value) + + if (data.numeric.is_memory) { + rewriteConfigBytesOption(state, name, value, data.numeric.default_value); + } else { + rewriteConfigNumericalOption(state, name, value, data.numeric.default_value); + } +} + +#define INTEGER_CONFIG 0 +#define MEMORY_CONFIG 1 + +#define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(numericConfigInit, numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \ + { .numeric = { \ + .is_memory = (memory), \ + .lower_bound = (lower), \ + .upper_bound = (upper), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), + +#define createIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_INT, \ + .config { .i = &(config_addr) } \ + } } \ +} + +#define createUIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_UINT, \ + .config { .ui = &(config_addr) } \ + } } \ +} + +#define createLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_LONG, \ + .config { .l = &(config_addr) } \ + } } \ +} + +#define createULongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG, \ + .config { .ul = &(config_addr) } \ + } } \ +} + +#define createLongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_LONG_LONG, \ + .config { .ll = &(config_addr) } \ + } } \ +} + +#define createULongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG_LONG, \ + .config { .ull = &(config_addr) } \ + } } \ +} + +#define createSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_SIZE_T, \ + .config { .st = &(config_addr) } \ + } } \ +} + +#define createSSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_SSIZE_T, \ + .config { .sst = &(config_addr) } \ + } } \ +} + +#define createTimeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_TIME_T, \ + .config { .tt = &(config_addr) } \ + } } \ +} + +#define createOffTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_OFF_T, \ + .config { .ot = &(config_addr) } \ + } } \ +} + +static int isValidActiveDefrag(int val, const char **err) { +#ifndef HAVE_DEFRAG + if (val) { + *err = "Active defragmentation cannot be enabled: it " + "requires a Redis server compiled with a modified Jemalloc " + "like the one shipped by default with the Redis source " + "distribution"; + return 0; + } +#else + UNUSED(val); + UNUSED(err); +#endif + return 1; +} + +static int isValidDBfilename(char *val, const char **err) { + if (!pathIsBaseName(val)) { + *err = "dbfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +static int isValidAOFfilename(char *val, const char **err) { + if (!pathIsBaseName(val)) { + *err = "appendfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +static int updateHZ(long long val, long long prev, const char **err) { + UNUSED(prev); + UNUSED(err); + /* Hz is more an hint from the user, so we accept values out of range + * but cap them to reasonable values. */ + g_pserver->config_hz = val; + if (g_pserver->config_hz < CONFIG_MIN_HZ) g_pserver->config_hz = CONFIG_MIN_HZ; + if (g_pserver->config_hz > CONFIG_MAX_HZ) g_pserver->config_hz = CONFIG_MAX_HZ; + g_pserver->hz = g_pserver->config_hz; + return 1; +} + +static int updateJemallocBgThread(int val, int prev, const char **err) { + UNUSED(prev); + UNUSED(err); + set_jemalloc_bg_thread(val); + return 1; +} + +static int updateReplBacklogSize(long long val, long long prev, const char **err) { + /* resizeReplicationBacklog sets g_pserver->repl_backlog_size, and relies on + * being able to tell when the size changes, so restore prev becore calling it. */ + UNUSED(err); + g_pserver->repl_backlog_size = prev; + resizeReplicationBacklog(val); + return 1; +} + +static int updateMaxmemory(long long val, long long prev, const char **err) { + UNUSED(prev); + UNUSED(err); + if (val) { + if ((unsigned long long)val < zmalloc_used_memory()) { + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); + } + freeMemoryIfNeededAndSafe(); + } + return 1; +} + +static int updateGoodSlaves(long long val, long long prev, const char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + refreshGoodSlavesCount(); + return 1; +} + +static int updateMasterAuthConfig(char *, char *, const char **) { + updateMasterAuth(); + return 1; +} + +static int updateAppendonly(int val, int prev, const char **err) { + UNUSED(prev); + if (val == 0 && g_pserver->aof_state != AOF_OFF) { + stopAppendOnly(); + } else if (val && g_pserver->aof_state == AOF_OFF) { + if (startAppendOnly() == C_ERR) { + *err = "Unable to turn on AOF. Check server logs."; + return 0; + } + } + return 1; +} + +static int updateMaxclients(long long val, long long prev, const char **err) { + /* Try to check if the OS is capable of supporting so many FDs. */ + if (val > prev) { + adjustOpenFilesLimit(); + if (g_pserver->maxclients != val) { + static char msg[128]; + sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", g_pserver->maxclients); + *err = msg; + if (g_pserver->maxclients > prev) { + g_pserver->maxclients = prev; + adjustOpenFilesLimit(); + } + return 0; + } + for (int iel = 0; iel < MAX_EVENT_LOOPS; ++iel) + { + if ((unsigned int) aeGetSetSize(g_pserver->rgthreadvar[iel].el) < + g_pserver->maxclients + CONFIG_FDSET_INCR) + { + if (aeResizeSetSize(g_pserver->rgthreadvar[iel].el, + g_pserver->maxclients + CONFIG_FDSET_INCR) == AE_ERR) + { + *err = "The event loop API used by Redis is not able to handle the specified number of clients"; + return 0; + } + } + } + } + return 1; +} + +#ifdef USE_OPENSSL +static int updateTlsCfg(char *val, char *prev, const char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + if (tlsConfigure(&g_pserver->tls_ctx_config) == C_ERR) { + *err = "Unable to configure tls-cert-file. Check server logs."; + return 0; + } + return 1; +} +static int updateTlsCfgBool(int val, int prev, const char **err) { + UNUSED(val); + UNUSED(prev); + return updateTlsCfg(NULL, NULL, err); +} +#endif /* USE_OPENSSL */ + +standardConfig configs[] = { + /* Bool configs */ + createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, g_pserver->rdb_checksum, 1, NULL, NULL), + createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, cserver.daemonize, 0, NULL, NULL), + createBoolConfig("lua-replicate-commands", NULL, MODIFIABLE_CONFIG, g_pserver->lua_always_replicate_commands, 1, NULL, NULL), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, g_pserver->always_show_logo, 0, NULL, NULL), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, g_pserver->protected_mode, 1, NULL, NULL), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, g_pserver->rdb_compression, 1, NULL, NULL), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, g_pserver->activerehashing, 1, NULL, NULL), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, g_pserver->stop_writes_on_bgsave_err, 1, NULL, NULL), + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, g_pserver->dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_eviction, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_expire, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_server_del, 0, NULL, NULL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, g_pserver->repl_disable_tcp_nodelay, 0, NULL, NULL), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, g_pserver->repl_diskless_sync, 0, NULL, NULL), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, g_pserver->aof_rewrite_incremental_fsync, 1, NULL, NULL), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, g_pserver->aof_no_fsync_on_rewrite, 0, NULL, NULL), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_require_full_coverage, 1, NULL, NULL), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, g_pserver->rdb_save_incremental_fsync, 1, NULL, NULL), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, g_pserver->aof_load_truncated, 1, NULL, NULL), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, g_pserver->aof_use_rdb_preamble, 1, NULL, NULL), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, g_pserver->cluster_slave_no_failover, 0, NULL, NULL), /* Failover by default. */ + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, g_pserver->repl_slave_lazy_flush, 0, NULL, NULL), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, g_pserver->repl_serve_stale_data, 1, NULL, NULL), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, g_pserver->repl_slave_ro, 1, NULL, NULL), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, g_pserver->repl_slave_ignore_maxmemory, 1, NULL, NULL), + createBoolConfig("multi-master", NULL, IMMUTABLE_CONFIG, g_pserver->enable_multimaster,CONFIG_DEFAULT_ENABLE_MULTIMASTER, NULL, NULL), + createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, cserver.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), + createBoolConfig("activedefrag", NULL, MODIFIABLE_CONFIG, cserver.active_defrag_enabled, 0, isValidActiveDefrag, NULL), + createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, g_pserver->syslog_enabled, 0, NULL, NULL), + createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, g_pserver->cluster_enabled, 0, NULL, NULL), + createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, g_pserver->aof_enabled, 0, NULL, updateAppendonly), + createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_reads_when_down, 0, NULL, NULL), + + /* String Configs */ + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->unixsocket, NULL, NULL, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.pidfile, NULL, NULL, NULL), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->slave_announce_ip, NULL, NULL, NULL), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masteruser, NULL, NULL, updateMasterAuthConfig), + createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masterauth, NULL, NULL, updateMasterAuthConfig), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->cluster_announce_ip, NULL, NULL, NULL), + createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->syslog_ident, "redis", NULL, NULL), + createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->rdb_filename, CONFIG_DEFAULT_RDB_FILENAME, isValidDBfilename, NULL), + createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->aof_filename, "appendonly.aof", isValidAOFfilename, NULL), + + /* Enum Configs */ + createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, cserver.supervised_mode, SUPERVISED_NONE, NULL, NULL), + createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, g_pserver->syslog_facility, LOG_LOCAL0, NULL, NULL), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, g_pserver->repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED, NULL, NULL), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, cserver.verbosity, LL_NOTICE, NULL, NULL), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, g_pserver->maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, g_pserver->aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), + + /* Integer configs */ + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, cserver.dbnum, 16, INTEGER_CONFIG, NULL, NULL), + createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, g_pserver->port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */ + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, g_pserver->list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, cserver.tcpkeepalive, 300, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->cluster_migration_barrier, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, cserver.active_defrag_cycle_min, 1, INTEGER_CONFIG, NULL, NULL), /* Default: 1% CPU min (at lower threshold) */ + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, cserver.active_defrag_cycle_max, 25, INTEGER_CONFIG, NULL, NULL), /* Default: 25% CPU max (at upper threshold) */ + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, cserver.active_defrag_threshold_lower, 10, INTEGER_CONFIG, NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, cserver.active_defrag_threshold_upper, 100, INTEGER_CONFIG, NULL, NULL), /* Default: maximum defrag force at 100% fragmentation */ + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->lfu_log_factor, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->lfu_decay_time, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->slave_priority, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, cserver.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */ + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, g_pserver->slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, g_pserver->tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use g_pserver->port */ + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, g_pserver->tracking_table_max_fill, 10, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max fill. */ + createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, cserver.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ + createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), + createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), + createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves), + /* Unsigned int configs */ + createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, g_pserver->maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients), + + /* Unsigned Long configs */ + createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, cserver.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ + createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + + /* Long Long configs */ + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, g_pserver->slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ + createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, g_pserver->repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + + /* Unsigned Long Long configs */ + createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), + + /* Size_t configs */ + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, cserver.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), + + /* Other configs */ + createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ + createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL), + +#ifdef USE_OPENSSL + createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, g_pserver->tls_cluster, 0, NULL, NULL), + createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, g_pserver->tls_replication, 0, NULL, NULL), + createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, g_pserver->tls_auth_clients, 1, NULL, NULL), + createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, g_pserver->tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool), + createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), + createStringConfig("tls-protocols", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.protocols, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphers", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ciphers, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphersuites", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ciphersuites, NULL, NULL, updateTlsCfg), +#endif + + /* NULL Terminator */ + {NULL} +}; + /*----------------------------------------------------------------------------- * CONFIG command entry point *----------------------------------------------------------------------------*/ diff --git a/src/connection.cpp b/src/connection.cpp new file mode 100644 index 000000000..b819c982a --- /dev/null +++ b/src/connection.cpp @@ -0,0 +1,425 @@ +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "connhelpers.h" + +/* The connections module provides a lean abstraction of network connections + * to avoid direct socket and async event management across the Redis code base. + * + * It does NOT provide advanced connection features commonly found in similar + * libraries such as complete in/out buffer management, throttling, etc. These + * functions remain in networking.c. + * + * The primary goal is to allow transparent handling of TCP and TLS based + * connections. To do so, connections have the following properties: + * + * 1. A connection may live before its corresponding socket exists. This + * allows various context and configuration setting to be handled before + * establishing the actual connection. + * 2. The caller may register/unregister logical read/write handlers to be + * called when the connection has data to read from/can accept writes. + * These logical handlers may or may not correspond to actual AE events, + * depending on the implementation (for TCP they are; for TLS they aren't). + */ + +extern ConnectionType CT_Socket; + +/* When a connection is created we must know its type already, but the + * underlying socket may or may not exist: + * + * - For accepted connections, it exists as we do not model the listen/accept + * part; So caller calls connCreateSocket() followed by connAccept(). + * - For outgoing connections, the socket is created by the connection module + * itself; So caller calls connCreateSocket() followed by connConnect(), + * which registers a connect callback that fires on connected/error state + * (and after any transport level handshake was done). + * + * NOTE: An earlier version relied on connections being part of other structs + * and not independently allocated. This could lead to further optimizations + * like using container_of(), etc. However it was discontinued in favor of + * this approach for these reasons: + * + * 1. In some cases conns are created/handled outside the context of the + * containing struct, in which case it gets a bit awkward to copy them. + * 2. Future implementations may wish to allocate arbitrary data for the + * connection. + * 3. The container_of() approach is anyway risky because connections may + * be embedded in different structs, not just client. + */ + +connection *connCreateSocket() { + connection *conn = (connection*)zcalloc(sizeof(connection), MALLOC_LOCAL); + conn->type = &CT_Socket; + conn->fd = -1; + + return conn; +} + +/* Create a new socket-type connection that is already associated with + * an accepted connection. + * + * The socket is not read for I/O until connAccept() was called and + * invoked the connection-level accept handler. + */ +connection *connCreateAcceptedSocket(int fd) { + connection *conn = connCreateSocket(); + conn->fd = fd; + conn->state = CONN_STATE_ACCEPTING; + return conn; +} + +static int connSocketConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + int fd = anetTcpNonBlockBestEffortBindConnect(NULL,addr,port,src_addr); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTING; + + conn->conn_handler = connect_handler; + aeCreateFileEvent(serverTL->el, conn->fd, AE_WRITABLE, + conn->type->ae_handler, conn); + + return C_OK; +} + +/* Returns true if a write handler is registered */ +int connHasWriteHandler(connection *conn) { + return conn->write_handler != NULL; +} + +/* Returns true if a read handler is registered */ +int connHasReadHandler(connection *conn) { + return conn->read_handler != NULL; +} + +/* Associate a private data pointer with the connection */ +void connSetPrivateData(connection *conn, void *data) { + conn->private_data = data; +} + +/* Get the associated private data pointer */ +void *connGetPrivateData(connection *conn) { + return conn->private_data; +} + +/* ------ Pure socket connections ------- */ + +/* A very incomplete list of implementation-specific calls. Much of the above shall + * move here as we implement additional connection types. + */ + +/* Close the connection and free resources. */ +static void connSocketClose(connection *conn) { + if (conn->fd != -1) { + aeDeleteFileEvent(serverTL->el,conn->fd,AE_READABLE); + aeDeleteFileEvent(serverTL->el,conn->fd,AE_WRITABLE); + close(conn->fd); + conn->fd = -1; + } + + /* If called from within a handler, schedule the close but + * keep the connection until the handler returns. + */ + if (conn->flags & CONN_FLAG_IN_HANDLER) { + conn->flags |= CONN_FLAG_CLOSE_SCHEDULED; + return; + } + + zfree(conn); +} + +static int connSocketWrite(connection *conn, const void *data, size_t data_len) { + int ret = write(conn->fd, data, data_len); + if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketRead(connection *conn, void *buf, size_t buf_len) { + int ret = read(conn->fd, buf, buf_len); + if (!ret) { + conn->state = CONN_STATE_CLOSED; + } else if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + if (conn->state != CONN_STATE_ACCEPTING) return C_ERR; + conn->state = CONN_STATE_CONNECTED; + if (!callHandler(conn, accept_handler)) return C_ERR; + return C_OK; +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + * + * The barrier flag indicates a write barrier is requested, resulting with + * CONN_FLAG_WRITE_BARRIER set. This will ensure that the write handler is + * always called before and not after the read handler in a single event + * loop. + */ +static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc func, int barrier, bool fThreadSafe) { + if (func == conn->write_handler) return C_OK; + + conn->write_handler = func; + if (barrier) + conn->flags |= CONN_FLAG_WRITE_BARRIER; + else + conn->flags &= ~CONN_FLAG_WRITE_BARRIER; + + int flags = AE_WRITABLE; + if (fThreadSafe) + flags |= AE_WRITE_THREADSAFE; + + if (!conn->write_handler) + aeDeleteFileEvent(serverTL->el,conn->fd,AE_WRITABLE); + else + if (aeCreateFileEvent(serverTL->el,conn->fd,flags, + conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static int connSocketSetReadHandler(connection *conn, ConnectionCallbackFunc func, bool fThreadSafe) { + if (func == conn->read_handler) return C_OK; + + int flags = AE_READABLE; + if (fThreadSafe) + flags |= AE_READ_THREADSAFE; + + conn->read_handler = func; + if (!conn->read_handler) + aeDeleteFileEvent(serverTL->el,conn->fd,AE_READABLE); + else + if (aeCreateFileEvent(serverTL->el,conn->fd, + flags,conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +static const char *connSocketGetLastError(connection *conn) { + return strerror(conn->last_errno); +} + +static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask) +{ + UNUSED(el); + UNUSED(fd); + connection *conn = (connection*)clientData; + + if (conn->state == CONN_STATE_CONNECTING && + (mask & AE_WRITABLE) && conn->conn_handler) { + + if (connGetSocketError(conn)) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } else { + conn->state = CONN_STATE_CONNECTED; + } + + if (!conn->write_handler) aeDeleteFileEvent(serverTL->el,conn->fd,AE_WRITABLE); + + if (!callHandler(conn, conn->conn_handler)) return; + conn->conn_handler = NULL; + } + + /* Normally we execute the readable event first, and the writable + * event later. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if WRITE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsync'ing a file to disk, + * before replying to a client. */ + int invert = conn->flags & CONN_FLAG_WRITE_BARRIER; + + int call_write = (mask & AE_WRITABLE) && conn->write_handler; + int call_read = (mask & AE_READABLE) && conn->read_handler; + + /* Handle normal I/O flows */ + if (!invert && call_read) { + if (!callHandler(conn, conn->read_handler)) return; + } + /* Fire the writable event. */ + if (call_write) { + if (!callHandler(conn, conn->write_handler)) return; + } + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert && call_read) { + if (!callHandler(conn, conn->read_handler)) return; + } +} + +static int connSocketBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + int fd = anetTcpNonBlockConnect(NULL,addr,port); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + if ((aeWait(fd, AE_WRITABLE, timeout) & AE_WRITABLE) == 0) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = ETIMEDOUT; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTED; + return C_OK; +} + +/* Connection-based versions of syncio.c functions. + * NOTE: This should ideally be refactored out in favor of pure async work. + */ + +static ssize_t connSocketSyncWrite(connection *conn, const char *ptr, ssize_t size, long long timeout) { + return syncWrite(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncRead(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncReadLine(conn->fd, ptr, size, timeout); +} + + +ConnectionType CT_Socket = { + .ae_handler = connSocketEventHandler, + .connect = connSocketConnect, + .write = connSocketWrite, + .read = connSocketRead, + .close = connSocketClose, + .accept = connSocketAccept, + .set_write_handler = connSocketSetWriteHandler, + .set_read_handler = connSocketSetReadHandler, + .get_last_error = connSocketGetLastError, + .blocking_connect = connSocketBlockingConnect, + .sync_write = connSocketSyncWrite, + .sync_read = connSocketSyncRead, + .sync_readline = connSocketSyncReadLine +}; + + +int connGetSocketError(connection *conn) { + int sockerr = 0; + socklen_t errlen = sizeof(sockerr); + + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) == -1) + sockerr = errno; + return sockerr; +} + +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port) { + return anetPeerToString(conn ? conn->fd : -1, ip, ip_len, port); +} + +int connFormatPeer(connection *conn, char *buf, size_t buf_len) { + return anetFormatPeer(conn ? conn->fd : -1, buf, buf_len); +} + +int connSockName(connection *conn, char *ip, size_t ip_len, int *port) { + return anetSockName(conn->fd, ip, ip_len, port); +} + +int connBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetBlock(NULL, conn->fd); +} + +int connNonBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetNonBlock(NULL, conn->fd); +} + +int connEnableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetEnableTcpNoDelay(NULL, conn->fd); +} + +int connDisableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetDisableTcpNoDelay(NULL, conn->fd); +} + +int connKeepAlive(connection *conn, int interval) { + if (conn->fd == -1) return C_ERR; + return anetKeepAlive(NULL, conn->fd, interval); +} + +int connSendTimeout(connection *conn, long long ms) { + return anetSendTimeout(NULL, conn->fd, ms); +} + +int connRecvTimeout(connection *conn, long long ms) { + return anetRecvTimeout(NULL, conn->fd, ms); +} + +int connGetState(connection *conn) { + return conn->state; +} + +void connSetThreadAffinity(connection *conn, int cpu) { +#ifdef HAVE_SO_INCOMING_CPU + if (setsockopt(conn->fd, SOL_SOCKET, SO_INCOMING_CPU, &cpu, sizeof(cpu)) != 0) + { + serverLog(LL_WARNING, "Failed to set socket affinity"); + } +#endif +} + +/* Return a text that describes the connection, suitable for inclusion + * in CLIENT LIST and similar outputs. + * + * For sockets, we always return "fd=" to maintain compatibility. + */ +const char *connGetInfo(connection *conn, char *buf, size_t buf_len) { + snprintf(buf, buf_len-1, "fd=%i", conn->fd); + return buf; +} + diff --git a/src/connection.h b/src/connection.h new file mode 100644 index 000000000..651df06bd --- /dev/null +++ b/src/connection.h @@ -0,0 +1,223 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNECTION_H +#define __REDIS_CONNECTION_H + +#define CONN_INFO_LEN 32 + +struct aeEventLoop; +typedef struct connection connection; + +typedef enum { + CONN_STATE_NONE = 0, + CONN_STATE_CONNECTING, + CONN_STATE_ACCEPTING, + CONN_STATE_CONNECTED, + CONN_STATE_CLOSED, + CONN_STATE_ERROR +} ConnectionState; + +#define CONN_FLAG_IN_HANDLER (1<<0) /* A handler execution is in progress */ +#define CONN_FLAG_CLOSE_SCHEDULED (1<<1) /* Closed scheduled by a handler */ +#define CONN_FLAG_WRITE_BARRIER (1<<2) /* Write barrier requested */ +#define CONN_FLAG_READ_THREADSAFE (1<<3) +#define CONN_FLAG_WRITE_THREADSAFE (1<<4) + +typedef void (*ConnectionCallbackFunc)(struct connection *conn); + +typedef struct ConnectionType { + void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask); + int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler); + int (*write)(struct connection *conn, const void *data, size_t data_len); + int (*read)(struct connection *conn, void *buf, size_t buf_len); + void (*close)(struct connection *conn); + int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); + int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler, int barrier, bool fThreadSafe); + int (*set_read_handler)(struct connection *conn, ConnectionCallbackFunc handler, bool fThreadSafe); + const char *(*get_last_error)(struct connection *conn); + int (*blocking_connect)(struct connection *conn, const char *addr, int port, long long timeout); + ssize_t (*sync_write)(struct connection *conn, const char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_read)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_readline)(struct connection *conn, char *ptr, ssize_t size, long long timeout); +} ConnectionType; + +struct connection { + ConnectionType *type; + ConnectionState state; + int flags; + int last_errno; + void *private_data; + ConnectionCallbackFunc conn_handler; + ConnectionCallbackFunc write_handler; + ConnectionCallbackFunc read_handler; + int fd; +}; + +/* The connection module does not deal with listening and accepting sockets, + * so we assume we have a socket when an incoming connection is created. + * + * The fd supplied should therefore be associated with an already accept()ed + * socket. + * + * connAccept() may directly call accept_handler(), or return and call it + * at a later time. This behavior is a bit awkward but aims to reduce the need + * to wait for the next event loop, if no additional handshake is required. + */ + +static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + return conn->type->accept(conn, accept_handler); +} + +/* Establish a connection. The connect_handler will be called when the connection + * is established, or if an error has occured. + * + * The connection handler will be responsible to set up any read/write handlers + * as needed. + * + * If C_ERR is returned, the operation failed and the connection handler shall + * not be expected. + */ +static inline int connConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + return conn->type->connect(conn, addr, port, src_addr, connect_handler); +} + +/* Blocking connect. + * + * NOTE: This is implemented in order to simplify the transition to the abstract + * connections, but should probably be refactored out of cluster.c and replication.c, + * in favor of a pure async implementation. + */ +static inline int connBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + return conn->type->blocking_connect(conn, addr, port, timeout); +} + +/* Write to connection, behaves the same as write(2). + * + * Like write(2), a short write is possible. A -1 return indicates an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connWrite(connection *conn, const void *data, size_t data_len) { + return conn->type->write(conn, data, data_len); +} + +/* Read from the connection, behaves the same as read(2). + * + * Like read(2), a short read is possible. A return value of 0 will indicate the + * connection was closed, and -1 will indicate an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connRead(connection *conn, void *buf, size_t buf_len) { + return conn->type->read(conn, buf, buf_len); +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + */ +static inline int connSetWriteHandler(connection *conn, ConnectionCallbackFunc func, bool fThreadSafe = false) { + return conn->type->set_write_handler(conn, func, 0, fThreadSafe); +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static inline int connSetReadHandler(connection *conn, ConnectionCallbackFunc func, bool fThreadSafe = false) { + return conn->type->set_read_handler(conn, func, fThreadSafe); +} + +/* Set a write handler, and possibly enable a write barrier, this flag is + * cleared when write handler is changed or removed. + * With barroer enabled, we never fire the event if the read handler already + * fired in the same event loop iteration. Useful when you want to persist + * things to disk before sending replies, and want to do that in a group fashion. */ +static inline int connSetWriteHandlerWithBarrier(connection *conn, ConnectionCallbackFunc func, int barrier, bool fThreadSafe = false) { + return conn->type->set_write_handler(conn, func, barrier, fThreadSafe); +} + +static inline void connClose(connection *conn) { + conn->type->close(conn); +} + +/* Returns the last error encountered by the connection, as a string. If no error, + * a NULL is returned. + */ +static inline const char *connGetLastError(connection *conn) { + return conn->type->get_last_error(conn); +} + +static inline ssize_t connSyncWrite(connection *conn, const char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_write(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_read(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_readline(conn, ptr, size, timeout); +} + +connection *connCreateSocket(); +connection *connCreateAcceptedSocket(int fd); + +connection *connCreateTLS(); +connection *connCreateAcceptedTLS(int fd, int require_auth); + +void connSetPrivateData(connection *conn, void *data); +void *connGetPrivateData(connection *conn); +int connGetState(connection *conn); +int connHasWriteHandler(connection *conn); +int connHasReadHandler(connection *conn); +int connGetSocketError(connection *conn); +void connSetThreadAffinity(connection *conn, int cpu); + +/* anet-style wrappers to conns */ +int connBlock(connection *conn); +int connNonBlock(connection *conn); +int connEnableTcpNoDelay(connection *conn); +int connDisableTcpNoDelay(connection *conn); +int connKeepAlive(connection *conn, int interval); +int connSendTimeout(connection *conn, long long ms); +int connRecvTimeout(connection *conn, long long ms); +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port); +int connFormatPeer(connection *conn, char *buf, size_t buf_len); +int connSockName(connection *conn, char *ip, size_t ip_len, int *port); +const char *connGetInfo(connection *conn, char *buf, size_t buf_len); + +/* Helpers for tls special considerations */ +int tlsHasPendingData(); +void tlsProcessPendingData(); + +#endif /* __REDIS_CONNECTION_H */ diff --git a/src/connhelpers.h b/src/connhelpers.h new file mode 100644 index 000000000..f237c9b1d --- /dev/null +++ b/src/connhelpers.h @@ -0,0 +1,85 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNHELPERS_H +#define __REDIS_CONNHELPERS_H + +#include "connection.h" + +/* These are helper functions that are common to different connection + * implementations (currently sockets in connection.c and TLS in tls.c). + * + * Currently helpers implement the mechanisms for invoking connection + * handlers, tracking in-handler states and dealing with deferred + * destruction (if invoked by a handler). + */ + +/* Called whenever a handler is invoked on a connection and sets the + * CONN_FLAG_IN_HANDLER flag to indicate we're in a handler context. + * + * An attempt to close a connection while CONN_FLAG_IN_HANDLER is + * set will result with deferred close, i.e. setting the CONN_FLAG_CLOSE_SCHEDULED + * instead of destructing it. + */ +static inline void enterHandler(connection *conn) { + conn->flags |= CONN_FLAG_IN_HANDLER; +} + +/* Called whenever a handler returns. This unsets the CONN_FLAG_IN_HANDLER + * flag and performs actual close/destruction if a deferred close was + * scheduled by the handler. + */ +static inline int exitHandler(connection *conn) { + conn->flags &= ~CONN_FLAG_IN_HANDLER; + if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { + connClose(conn); + return 0; + } + return 1; +} + +/* Helper for connection implementations to call handlers: + * 1. Mark the handler in use. + * 2. Execute the handler (if set). + * 3. Mark the handler as NOT in use and perform deferred close if was + * requested by the handler at any time. + */ +static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) { + conn->flags |= CONN_FLAG_IN_HANDLER; + if (handler) handler(conn); + conn->flags &= ~CONN_FLAG_IN_HANDLER; + if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { + connClose(conn); + return 0; + } + return 1; +} + +#endif /* __REDIS_CONNHELPERS_H */ diff --git a/src/cron.cpp b/src/cron.cpp index fd777d43d..2c20a5c0e 100644 --- a/src/cron.cpp +++ b/src/cron.cpp @@ -77,7 +77,7 @@ void executeCronJobExpireHook(const char *key, robj *o) serverAssert(o->type == OBJ_CRON); cronjob *job = (cronjob*)ptrFromObj(o); - client *cFake = createClient(-1, IDX_EVENT_LOOP_MAIN); + client *cFake = createClient(nullptr, IDX_EVENT_LOOP_MAIN); cFake->lock.lock(); cFake->authenticated = 1; cFake->puser = nullptr; diff --git a/src/db.cpp b/src/db.cpp index 605baaf40..bff6425cc 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -77,10 +77,7 @@ static robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (g_pserver->rdb_child_pid == -1 && - g_pserver->aof_child_pid == -1 && - !(flags & LOOKUP_NOTOUCH)) - { + if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)){ if (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { @@ -176,13 +173,17 @@ robj_roptr lookupKeyRead(redisDb *db, robj *key) { * * Returns the linked value object if the key exists or NULL if the key * does not exist in the specified DB. */ -robj *lookupKeyWrite(redisDb *db, robj *key) { - robj *o = lookupKey(db,key,LOOKUP_UPDATEMVCC); +robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { + robj *o = lookupKey(db,key,flags|LOOKUP_UPDATEMVCC); if (expireIfNeeded(db,key)) - o = NULL; + o = nullptr; return o; } +robj *lookupKeyWrite(redisDb *db, robj *key) { + return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); +} + robj_roptr lookupKeyReadOrReply(client *c, robj *key, robj *reply) { robj_roptr o = lookupKeyRead(c->db, key); if (!o) addReply(c,reply); @@ -302,20 +303,28 @@ int dbMerge(redisDb *db, robj *key, robj *val, int fReplace) * * 1) The ref count of the value object is incremented. * 2) clients WATCHing for the destination key notified. - * 3) The expire time of the key is reset (the key is made persistent). + * 3) The expire time of the key is reset (the key is made persistent), + * unless 'keepttl' is true. * * All the new keys in the database should be created via this interface. */ -void setKey(redisDb *db, robj *key, robj *val) { +void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl) { dictEntry *de = dictFind(db->pdict, ptrFromObj(key)); if (de == NULL) { dbAdd(db,key,val); } else { - dbOverwriteCore(db,de,key,val,!!g_pserver->fActiveReplica,true); + dbOverwriteCore(db,de,key,val,!!g_pserver->fActiveReplica,!keepttl); } incrRefCount(val); signalModifiedKey(db,key); } +/* Common case for genericSetKey() where the TTL is not retained. */ +void setKey(redisDb *db, robj *key, robj *val) { + genericSetKey(db,key,val,0); +} + +/* Return true if the specified key exists in the specified database. + * LRU/LFU info is not updated in any way. */ int dbExists(redisDb *db, robj *key) { return dictFind(db->pdict,ptrFromObj(key)) != NULL; } @@ -441,7 +450,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * On success the fuction returns the number of keys removed from the * database(s). Otherwise -1 is returned in the specific case the * DB number is out of range, and errno is set to EINVAL. */ -long long emptyDb(int dbnum, int flags, void(callback)(void*)) { +long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) { int async = (flags & EMPTYDB_ASYNC); long long removed = 0; @@ -450,6 +459,17 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { return -1; } + /* Fire the flushdb modules event. */ + RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_START, + &fi); + + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum); + int startdb, enddb; if (dbnum == -1) { startdb = 0; @@ -459,14 +479,12 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { } for (int j = startdb; j <= enddb; j++) { - removed += dictSize(g_pserver->db[j].pdict); + removed += dictSize(dbarray[j].pdict); if (async) { - emptyDbAsync(&g_pserver->db[j]); + emptyDbAsync(&dbarray[j]); } else { - dictEmpty(g_pserver->db[j].pdict,callback); - delete g_pserver->db[j].setexpire; - g_pserver->db[j].setexpire = new (MALLOC_LOCAL) expireset(); - g_pserver->db[j].expireitr = g_pserver->db[j].setexpire->end(); + dictEmpty(dbarray[j].pdict,callback); + dbarray[j].setexpire->clear(); } } if (g_pserver->cluster_enabled) { @@ -477,9 +495,20 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { } } if (dbnum == -1) flushSlaveKeysWithExpireList(); + + /* Also fire the end event. Note that this event will fire almost + * immediately after the start event if the flush is asynchronous. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_END, + &fi); + return removed; } +long long emptyDb(int dbnum, int flags, void(callback)(void*)) { + return emptyDbGeneric(g_pserver->db, dbnum, flags, callback); +} + int selectDb(client *c, int id) { if (id < 0 || id >= cserver.dbnum) return C_ERR; @@ -487,6 +516,15 @@ int selectDb(client *c, int id) { return C_OK; } +long long dbTotalServerKeyCount() { + long long total = 0; + int j; + for (j = 0; j < cserver.dbnum; j++) { + total += dictSize(g_pserver->db[j].pdict); + } + return total; +} + /*----------------------------------------------------------------------------- * Hooks for key space changes. * @@ -498,11 +536,12 @@ int selectDb(client *c, int id) { void signalModifiedKey(redisDb *db, robj *key) { touchWatchedKey(db,key); - if (g_pserver->tracking_clients) trackingInvalidateKey(key); + trackingInvalidateKey(key); } void signalFlushedDb(int dbid) { touchWatchedKeysOnFlush(dbid); + trackingInvalidateKeysOnFlush(dbid); } /*----------------------------------------------------------------------------- @@ -531,28 +570,9 @@ int getFlushCommandFlags(client *c, int *flags) { return C_OK; } -/* FLUSHDB [ASYNC] - * - * Flushes the currently SELECTed Redis DB. */ -void flushdbCommand(client *c) { - int flags; - - if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(c->db->id); - g_pserver->dirty += emptyDb(c->db->id,flags,NULL); - addReply(c,shared.ok); -} - -/* FLUSHALL [ASYNC] - * - * Flushes the whole server data set. */ -void flushallCommand(client *c) { - int flags; - - if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(-1); +/* Flushes the whole server data set. */ +void flushAllDataAndResetRDB(int flags) { g_pserver->dirty += emptyDb(-1,flags,NULL); - addReply(c,shared.ok); if (g_pserver->rdb_child_pid != -1) killRDBChild(); if (g_pserver->saveparamslen > 0) { /* Normally rdbSave() will reset dirty, but we don't want this here @@ -564,6 +584,41 @@ void flushallCommand(client *c) { g_pserver->dirty = saved_dirty; } g_pserver->dirty++; +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif +} + +/* FLUSHDB [ASYNC] + * + * Flushes the currently SELECTed Redis DB. */ +void flushdbCommand(client *c) { + int flags; + + if (getFlushCommandFlags(c,&flags) == C_ERR) return; + g_pserver->dirty += emptyDb(c->db->id,flags,NULL); + addReply(c,shared.ok); +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif +} + +/* FLUSHALL [ASYNC] + * + * Flushes the whole server data set. */ +void flushallCommand(client *c) { + int flags; + if (getFlushCommandFlags(c,&flags) == C_ERR) return; + flushAllDataAndResetRDB(flags); + addReply(c,shared.ok); } /* This command implements DEL and LAZYDEL. */ @@ -644,7 +699,7 @@ void keysCommand(client *c) { void *replylen = addReplyDeferredLen(c); di = dictGetSafeIterator(c->db->pdict); - allkeys = (pattern[0] == '*' && pattern[1] == '\0'); + allkeys = (pattern[0] == '*' && plen == 1); while((de = dictNext(di)) != NULL) { sds key = (sds)dictGetKey(de); robj *keyobj; @@ -1100,6 +1155,13 @@ void moveCommand(client *c) { dbAdd(dst,c->argv[1],o); if (spexpire != nullptr) setExpire(c,dst,c->argv[1],std::move(*spexpire)); + signalModifiedKey(src,c->argv[1]); + signalModifiedKey(dst,c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_from",c->argv[1],src->id); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_to",c->argv[1],dst->id); + addReply(c,shared.cone); } @@ -1129,7 +1191,7 @@ void scanDatabaseForReadyLists(redisDb *db) { * * Returns C_ERR if at least one of the DB ids are out of range, otherwise * C_OK is returned. */ -int dbSwapDatabases(int id1, int id2) { +int dbSwapDatabases(long id1, long id2) { if (id1 < 0 || id1 >= cserver.dbnum || id2 < 0 || id2 >= cserver.dbnum) return C_ERR; if (id1 == id2) return C_OK; @@ -1385,6 +1447,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { /* Check if the key is expired. Note, this does not check subexpires */ int keyIsExpired(redisDb *db, robj *key) { expireEntry *pexpire = getExpire(db,key); + mstime_t now; if (pexpire == nullptr) return 0; /* No expire for this key */ @@ -1409,8 +1472,26 @@ int keyIsExpired(redisDb *db, robj *key) { * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. * See issue #1525 on Github for more information. */ - mstime_t now = g_pserver->lua_caller ? g_pserver->lua_time_start : mstime(); + if (g_pserver->lua_caller) { + now = g_pserver->lua_time_start; + } + /* If we are in the middle of a command execution, we still want to use + * a reference time that does not change: in that case we just use the + * cached time, that we update before each call in the call() function. + * This way we avoid that commands such as RPOPLPUSH or similar, that + * may re-open the same key multiple times, can invalidate an already + * open object in a next call, if the next call will see the key expired, + * while the first did not. */ + else if (serverTL->fixed_time_expire > 0) { + now = g_pserver->mstime; + } + /* For the other cases, we want to use the most fresh time we have. */ + else { + now = mstime(); + } + /* The key expired if the current (virtual or real) time is greater + * than the expire time of the key. */ return now > when; } diff --git a/src/debug.cpp b/src/debug.cpp index 234f197be..a65217bb3 100644 --- a/src/debug.cpp +++ b/src/debug.cpp @@ -306,6 +306,56 @@ void computeDatasetDigest(unsigned char *final) { } } +#ifdef USE_JEMALLOC +void mallctl_int(client *c, robj **argv, int argc) { + int ret; + /* start with the biggest size (int64), and if that fails, try smaller sizes (int32, bool) */ + int64_t old = 0, val; + if (argc > 1) { + long long ll; + if (getLongLongFromObjectOrReply(c, argv[1], &ll, NULL) != C_OK) + return; + val = ll; + } + size_t sz = sizeof(old); + while (sz > 0) { + if ((ret=je_mallctl(szFromObj(argv[0]), &old, &sz, argc > 1? &val: NULL, argc > 1?sz: 0))) { + if (ret==EINVAL) { + /* size might be wrong, try a smaller one */ + sz /= 2; +#if BYTE_ORDER == BIG_ENDIAN + val <<= 8*sz; +#endif + continue; + } + addReplyErrorFormat(c,"%s", strerror(ret)); + return; + } else { +#if BYTE_ORDER == BIG_ENDIAN + old >>= 64 - 8*sz; +#endif + addReplyLongLong(c, old); + return; + } + } + addReplyErrorFormat(c,"%s", strerror(EINVAL)); +} + +void mallctl_string(client *c, robj **argv, int argc) { + int ret; + char *old; + size_t sz = sizeof(old); + /* for strings, it seems we need to first get the old value, before overriding it. */ + if ((ret=je_mallctl(szFromObj(argv[0]), &old, &sz, NULL, 0))) { + addReplyErrorFormat(c,"%s", strerror(ret)); + return; + } + addReplyBulkCString(c, old); + if(argc > 1) + je_mallctl(szFromObj(argv[0]), NULL, 0, &argv[1]->m_ptr, sizeof(char*)); +} +#endif + void debugCommand(client *c) { if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"help")) { const char *help[] = { @@ -328,10 +378,15 @@ void debugCommand(client *c) { "SDSLEN -- Show low level SDS string info representing key and value.", "SEGFAULT -- Crash the server with sigsegv.", "SET-ACTIVE-EXPIRE <0|1> -- Setting it to 0 disables expiring keys in background when they are not accessed (otherwise the Redis behavior). Setting it to 1 reenables back the default.", +"AOF-FLUSH-SLEEP -- Server will sleep before flushing the AOF, this is used for testing", "SLEEP -- Stop the server for . Decimals allowed.", "STRUCTSIZE -- Return the size of different Redis core C structures.", "ZIPLIST -- Show low level info about the ziplist encoding.", "STRINGMATCH-TEST -- Run a fuzz tester against the stringmatchlen() function.", +#ifdef USE_JEMALLOC +"MALLCTL [] -- Get or set a malloc tunning integer.", +"MALLCTL-STR [] -- Get or set a malloc tunning string.", +#endif NULL }; addReplyHelp(c, help); @@ -371,8 +426,7 @@ NULL } emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); protectClient(c); - rdbSaveInfo rsiDft = RDB_SAVE_INFO_INIT; - int ret = rdbLoad(&rsiDft); + int ret = rdbLoadFile(g_pserver->rdb_filename,NULL,RDBFLAGS_NONE); unprotectClient(c); if (ret != C_OK) { addReplyError(c,"Error trying to load the RDB dump"); @@ -605,6 +659,11 @@ NULL { g_pserver->active_expire_enabled = atoi(szFromObj(c->argv[2])); addReply(c,shared.ok); + } else if (!strcasecmp(szFromObj(c->argv[1]),"aof-flush-sleep") && + c->argc == 3) + { + g_pserver->aof_flush_sleep = atoi(szFromObj(c->argv[2])); + addReply(c,shared.ok); } else if (!strcasecmp(szFromObj(c->argv[1]),"lua-always-replicate-commands") && c->argc == 3) { @@ -648,7 +707,8 @@ NULL g_pserver->db[dbid].setexpire->getstats(buf, sizeof(buf)); stats = sdscat(stats, buf); - addReplyBulkSds(c,stats); + addReplyVerbatim(c,stats,sdslen(stats),"txt"); + sdsfree(stats); } else if (!strcasecmp(szFromObj(c->argv[1]),"htstats-key") && c->argc == 3) { robj *o; dict *ht = NULL; @@ -675,7 +735,7 @@ NULL } else { char buf[4096]; dictGetStats(buf,sizeof(buf),ht); - addReplyBulkCString(c,buf); + addReplyVerbatim(c,buf,strlen(buf),"txt"); } } else if (!strcasecmp(szFromObj(c->argv[1]),"change-repl-id") && c->argc == 2) { serverLog(LL_WARNING,"Changing replication IDs after receiving DEBUG change-repl-id"); @@ -688,6 +748,14 @@ NULL } else if (!strcasecmp(szFromObj(c->argv[1]), "force-master") && c->argc == 2) { c->flags |= CLIENT_MASTER | CLIENT_MASTER_FORCE_REPLY; addReply(c, shared.ok); +#ifdef USE_JEMALLOC + } else if(!strcasecmp(szFromObj(c->argv[1]),"mallctl") && c->argc >= 3) { + mallctl_int(c, c->argv+2, c->argc-2); + return; + } else if(!strcasecmp(szFromObj(c->argv[1]),"mallctl-str") && c->argc >= 3) { + mallctl_string(c, c->argv+2, c->argc-2); + return; +#endif } else { addReplySubcommandSyntaxError(c); return; @@ -711,11 +779,12 @@ void _serverAssert(const char *estr, const char *file, int line) { void _serverAssertPrintClientInfo(const client *c) { int j; + char conninfo[CONN_INFO_LEN]; bugReportStart(); serverLog(LL_WARNING,"=== ASSERTION FAILED CLIENT CONTEXT ==="); - serverLog(LL_WARNING,"client->flags = %llu", static_cast(c->flags)); - serverLog(LL_WARNING,"client->fd = %d", c->fd); + serverLog(LL_WARNING,"client->flags = %llu", (unsigned long long) c->flags); + serverLog(LL_WARNING,"client->conn = %s", connGetInfo(c->conn, conninfo, sizeof(conninfo))); serverLog(LL_WARNING,"client->argc = %d", c->argc); for (j=0; j < c->argc; j++) { char buf[128]; @@ -1122,6 +1191,33 @@ void logRegisters(ucontext_t *uc) { (unsigned long) uc->uc_mcontext.mc_cs ); logStackContent((void**)uc->uc_mcontext.mc_rsp); +#elif defined(__aarch64__) /* Linux AArch64 */ + serverLog(LL_WARNING, + "\n" + "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n" + "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n" + "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n" + "X30:%016lx\n" + "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.regs[18], + (unsigned long) uc->uc_mcontext.regs[19], + (unsigned long) uc->uc_mcontext.regs[20], + (unsigned long) uc->uc_mcontext.regs[21], + (unsigned long) uc->uc_mcontext.regs[22], + (unsigned long) uc->uc_mcontext.regs[23], + (unsigned long) uc->uc_mcontext.regs[24], + (unsigned long) uc->uc_mcontext.regs[25], + (unsigned long) uc->uc_mcontext.regs[26], + (unsigned long) uc->uc_mcontext.regs[27], + (unsigned long) uc->uc_mcontext.regs[28], + (unsigned long) uc->uc_mcontext.regs[29], + (unsigned long) uc->uc_mcontext.regs[30], + (unsigned long) uc->uc_mcontext.pc, + (unsigned long) uc->uc_mcontext.sp, + (unsigned long) uc->uc_mcontext.pstate, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.sp); #else serverLog(LL_WARNING, " Dumping of registers not supported for this OS/arch"); @@ -1405,6 +1501,12 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { /* Log dump of processor registers */ logRegisters(uc); + /* Log Modules INFO */ + serverLogRaw(LL_WARNING|LL_RAW, "\n------ MODULES INFO OUTPUT ------\n"); + infostring = modulesCollectInfo(sdsempty(), NULL, 1, 0); + serverLogRaw(LL_WARNING|LL_RAW, infostring); + sdsfree(infostring); + #if defined(HAVE_PROC_MAPS) /* Test memory */ serverLogRaw(LL_WARNING|LL_RAW, "\n------ FAST MEMORY TEST ------\n"); diff --git a/src/defrag.cpp b/src/defrag.cpp index 1b5596609..97297b8e5 100644 --- a/src/defrag.cpp +++ b/src/defrag.cpp @@ -373,7 +373,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { if ((newele = activeDefragStringOb(ele, &defragged))) de->v.val = newele, defragged++; } else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) { - void *newptr, *ptr = ln->value; + void *newptr, *ptr = dictGetVal(de); if ((newptr = activeDefragAlloc(ptr))) ln->value = newptr, defragged++; } @@ -935,10 +935,12 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) { return 0; } +/* static variables serving defragLaterStep to continue scanning a key from were we stopped last time. */ +static sds defrag_later_current_key = NULL; +static unsigned long defrag_later_cursor = 0; + /* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */ int defragLaterStep(redisDb *db, long long endtime) { - static sds current_key = NULL; - static unsigned long cursor = 0; unsigned int iterations = 0; unsigned long long prev_defragged = g_pserver->stat_active_defrag_hits; unsigned long long prev_scanned = g_pserver->stat_active_defrag_scanned; @@ -946,16 +948,15 @@ int defragLaterStep(redisDb *db, long long endtime) { do { /* if we're not continuing a scan from the last call or loop, start a new one */ - if (!cursor) { + if (!defrag_later_cursor) { listNode *head = listFirst(db->defrag_later); /* Move on to next key */ - if (current_key) { - serverAssert(current_key == head->value); - sdsfree((sds)head->value); + if (defrag_later_current_key) { + serverAssert(defrag_later_current_key == head->value); listDelNode(db->defrag_later, head); - cursor = 0; - current_key = NULL; + defrag_later_cursor = 0; + defrag_later_current_key = NULL; } /* stop if we reached the last one. */ @@ -964,21 +965,21 @@ int defragLaterStep(redisDb *db, long long endtime) { return 0; /* start a new key */ - current_key = (sds)head->value; - cursor = 0; + defrag_later_current_key = (sds)head->value; + defrag_later_cursor = 0; } /* each time we enter this function we need to fetch the key from the dict again (if it still exists) */ - dictEntry *de = dictFind(db->pdict, current_key); + dictEntry *de = dictFind(db->pdict, defrag_later_current_key); key_defragged = g_pserver->stat_active_defrag_hits; do { int quit = 0; - if (defragLaterItem(de, &cursor, endtime)) + if (defragLaterItem(de, &defrag_later_cursor, endtime)) quit = 1; /* time is up, we didn't finish all the work */ /* Don't start a new BIG key in this loop, this is because the * next key can be a list, and scanLaterList must be done in once cycle */ - if (!cursor) + if (!defrag_later_cursor) quit = 1; /* Once in 16 scan iterations, 512 pointer reallocations, or 64 fields @@ -998,7 +999,7 @@ int defragLaterStep(redisDb *db, long long endtime) { prev_defragged = g_pserver->stat_active_defrag_hits; prev_scanned = g_pserver->stat_active_defrag_scanned; } - } while(cursor); + } while(defrag_later_cursor); if(key_defragged != g_pserver->stat_active_defrag_hits) g_pserver->stat_active_defrag_key_hits++; else @@ -1055,7 +1056,22 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (g_pserver->aof_child_pid!=-1 || g_pserver->rdb_child_pid!=-1) + if (!cserver.active_defrag_enabled) { + if (g_pserver->active_defrag_running) { + /* if active defrag was disabled mid-run, start from fresh next time. */ + g_pserver->active_defrag_running = 0; + if (db) + listEmpty(db->defrag_later); + defrag_later_current_key = NULL; + defrag_later_cursor = 0; + current_db = -1; + cursor = 0; + db = NULL; + } + return; + } + + if (hasActiveChildProcess()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/src/evict.cpp b/src/evict.cpp index 04fd7a7d6..7b666523b 100644 --- a/src/evict.cpp +++ b/src/evict.cpp @@ -460,6 +460,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev * was freed to return back under the limit, the function returns C_ERR. */ int freeMemoryIfNeeded(void) { serverAssert(GlobalLocksAcquired()); + int keys_freed = 0; /* By default replicas should ignore maxmemory * and just be masters exact copies. */ if (listLength(g_pserver->masters) && g_pserver->repl_slave_ignore_maxmemory) return C_OK; @@ -483,7 +484,7 @@ int freeMemoryIfNeeded(void) { latencyStartMonitor(latency); while (mem_freed < mem_tofree) { - int j, k, i, keys_freed = 0; + int j, k, i; static unsigned int next_db = 0; sds bestkey = NULL; int bestdbid; @@ -627,9 +628,7 @@ int freeMemoryIfNeeded(void) { mem_freed = mem_tofree; } } - } - - if (!keys_freed) { + } else { latencyEndMonitor(latency); latencyAddSampleIfNeeded("eviction-cycle",latency); goto cant_free; /* nothing to free... */ diff --git a/src/geo.cpp b/src/geo.cpp index 55c52fc88..9a84b821f 100644 --- a/src/geo.cpp +++ b/src/geo.cpp @@ -465,8 +465,8 @@ void georadiusGeneric(client *c, int flags) { int storedist = 0; /* 0 for STORE, 1 for STOREDIST. */ /* Look up the requested zset */ - robj_roptr zobj; - if ((zobj = lookupKeyReadOrReply(c, key, shared.null[c->resp])) == nullptr || + robj_roptr zobj = nullptr; + if ((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == nullptr || checkType(c, zobj, OBJ_ZSET)) { return; } @@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) { /* If no matching results, the user gets an empty reply. */ if (ga->used == 0 && storekey == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); geoArrayFree(ga); return; } @@ -737,7 +737,15 @@ void geohashCommand(client *c) { char buf[12]; int i; for (i = 0; i < 11; i++) { - int idx = (hash.bits >> (52-((i+1)*5))) & 0x1f; + int idx; + if (i == 10) { + /* We have just 52 bits, but the API used to output + * an 11 bytes geohash. For compatibility we assume + * zero. */ + idx = 0; + } else { + idx = (hash.bits >> (52-((i+1)*5))) & 0x1f; + } buf[i] = geoalphabet[idx]; } buf[11] = '\0'; diff --git a/src/hyperloglog.cpp b/src/hyperloglog.cpp index 0b1239965..6cbd7b94e 100644 --- a/src/hyperloglog.cpp +++ b/src/hyperloglog.cpp @@ -709,8 +709,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) { p += oplen; first += span; } - if (span == 0) return -1; /* Invalid format. */ - if (p >= end) return -1; /* Invalid format. */ + if (span == 0 || p >= end) return -1; /* Invalid format. */ next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; if (next >= end) next = NULL; @@ -1253,7 +1252,7 @@ void pfcountCommand(client *c) { if (o == nullptr) continue; /* Assume empty HLL for non existing var.*/ if (isHLLObjectOrReply(c,o) != C_OK) return; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(registers,HLL_REGISTERS,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); @@ -1340,7 +1339,7 @@ void pfmergeCommand(client *c) { hdr = (hllhdr*)ptrFromObj(o); if (hdr->encoding == HLL_DENSE) use_dense = 1; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(max,sizeof(max),o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); diff --git a/src/latency.cpp b/src/latency.cpp index 0dd751a3a..3cc599dd6 100644 --- a/src/latency.cpp +++ b/src/latency.cpp @@ -599,7 +599,7 @@ NULL event = (char*)dictGetKey(de); graph = latencyCommandGenSparkeline(event,ts); - addReplyBulkCString(c,graph); + addReplyVerbatim(c,graph,sdslen(graph),"txt"); sdsfree(graph); } else if (!strcasecmp(szFromObj(c->argv[1]),"latest") && c->argc == 2) { /* LATENCY LATEST */ @@ -608,7 +608,7 @@ NULL /* LATENCY DOCTOR */ sds report = createLatencyReport(); - addReplyBulkCBuffer(c,report,sdslen(report)); + addReplyVerbatim(c,report,sdslen(report),"txt"); sdsfree(report); } else if (!strcasecmp(szFromObj(c->argv[1]),"reset") && c->argc >= 2) { /* LATENCY RESET */ diff --git a/src/localtime.c b/src/localtime.c index 3f59a3331..e2ac81f98 100644 --- a/src/localtime.c +++ b/src/localtime.c @@ -52,8 +52,8 @@ static int is_leap_year(time_t year) { if (year % 4) return 0; /* A year not divisible by 4 is not leap. */ else if (year % 100) return 1; /* If div by 4 and not 100 is surely leap. */ - else if (year % 400) return 0; /* If div by 100 *and* 400 is not leap. */ - else return 1; /* If div by 100 and not by 400 is leap. */ + else if (year % 400) return 0; /* If div by 100 *and* not by 400 is not leap. */ + else return 1; /* If div by 100 and 400 is leap. */ } void nolocks_localtime(struct tm *tmp, time_t t, time_t tz, int dst) { diff --git a/src/lolwut.c b/src/lolwut.c new file mode 100644 index 000000000..0e1552ba0 --- /dev/null +++ b/src/lolwut.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2018, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ---------------------------------------------------------------------------- + * + * This file implements the LOLWUT command. The command should do something + * fun and interesting, and should be replaced by a new implementation at + * each new version of Redis. + */ + +#include "server.h" +#include "lolwut.h" +#include + +void lolwut5Command(client *c); +void lolwut6Command(client *c); + +/* The default target for LOLWUT if no matching version was found. + * This is what unstable versions of Redis will display. */ +void lolwutUnstableCommand(client *c) { + sds rendered = sdsnew("Redis ver. "); + rendered = sdscat(rendered,REDIS_VERSION); + rendered = sdscatlen(rendered,"\n",1); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); +} + +/* LOLWUT [VERSION ] [... version specific arguments ...] */ +void lolwutCommand(client *c) { + char *v = REDIS_VERSION; + char verstr[64]; + + if (c->argc >= 3 && !strcasecmp(c->argv[1]->ptr,"version")) { + long ver; + if (getLongFromObjectOrReply(c,c->argv[2],&ver,NULL) != C_OK) return; + snprintf(verstr,sizeof(verstr),"%u.0.0",(unsigned int)ver); + v = verstr; + + /* Adjust argv/argc to filter the "VERSION ..." option, since the + * specific LOLWUT version implementations don't know about it + * and expect their arguments. */ + c->argv += 2; + c->argc -= 2; + } + + if ((v[0] == '5' && v[1] == '.' && v[2] != '9') || + (v[0] == '4' && v[1] == '.' && v[2] == '9')) + lolwut5Command(c); + else if ((v[0] == '6' && v[1] == '.' && v[2] != '9') || + (v[0] == '5' && v[1] == '.' && v[2] == '9')) + lolwut6Command(c); + else + lolwutUnstableCommand(c); + + /* Fix back argc/argv in case of VERSION argument. */ + if (v == verstr) { + c->argv -= 2; + c->argc += 2; + } +} + +/* ========================== LOLWUT Canvase =============================== + * Many LOWUT versions will likely print some computer art to the screen. + * This is the case with LOLWUT 5 and LOLWUT 6, so here there is a generic + * canvas implementation that can be reused. */ + +/* Allocate and return a new canvas of the specified size. */ +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor) { + lwCanvas *canvas = zmalloc(sizeof(*canvas)); + canvas->width = width; + canvas->height = height; + canvas->pixels = zmalloc(width*height); + memset(canvas->pixels,bgcolor,width*height); + return canvas; +} + +/* Free the canvas created by lwCreateCanvas(). */ +void lwFreeCanvas(lwCanvas *canvas) { + zfree(canvas->pixels); + zfree(canvas); +} + +/* Set a pixel to the specified color. Color is 0 or 1, where zero means no + * dot will be displyed, and 1 means dot will be displayed. + * Coordinates are arranged so that left-top corner is 0,0. You can write + * out of the size of the canvas without issues. */ +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return; + canvas->pixels[x+y*canvas->width] = color; +} + +/* Return the value of the specified pixel on the canvas. */ +int lwGetPixel(lwCanvas *canvas, int x, int y) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return 0; + return canvas->pixels[x+y*canvas->width]; +} + +/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */ +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) { + int dx = abs(x2-x1); + int dy = abs(y2-y1); + int sx = (x1 < x2) ? 1 : -1; + int sy = (y1 < y2) ? 1 : -1; + int err = dx-dy, e2; + + while(1) { + lwDrawPixel(canvas,x1,y1,color); + if (x1 == x2 && y1 == y2) break; + e2 = err*2; + if (e2 > -dy) { + err -= dy; + x1 += sx; + } + if (e2 < dx) { + err += dx; + y1 += sy; + } + } +} + +/* Draw a square centered at the specified x,y coordinates, with the specified + * rotation angle and size. In order to write a rotated square, we use the + * trivial fact that the parametric equation: + * + * x = sin(k) + * y = cos(k) + * + * Describes a circle for values going from 0 to 2*PI. So basically if we start + * at 45 degrees, that is k = PI/4, with the first point, and then we find + * the other three points incrementing K by PI/2 (90 degrees), we'll have the + * points of the square. In order to rotate the square, we just start with + * k = PI/4 + rotation_angle, and we are done. + * + * Of course the vanilla equations above will describe the square inside a + * circle of radius 1, so in order to draw larger squares we'll have to + * multiply the obtained coordinates, and then translate them. However this + * is much simpler than implementing the abstract concept of 2D shape and then + * performing the rotation/translation transformation, so for LOLWUT it's + * a good approach. */ +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color) { + int px[4], py[4]; + + /* Adjust the desired size according to the fact that the square inscribed + * into a circle of radius 1 has the side of length SQRT(2). This way + * size becomes a simple multiplication factor we can use with our + * coordinates to magnify them. */ + size /= 1.4142135623; + size = round(size); + + /* Compute the four points. */ + float k = M_PI/4 + angle; + for (int j = 0; j < 4; j++) { + px[j] = round(sin(k) * size + x); + py[j] = round(cos(k) * size + y); + k += M_PI/2; + } + + /* Draw the square. */ + for (int j = 0; j < 4; j++) + lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],color); +} diff --git a/src/lolwut.h b/src/lolwut.h new file mode 100644 index 000000000..38c0de423 --- /dev/null +++ b/src/lolwut.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2018-2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* This structure represents our canvas. Drawing functions will take a pointer + * to a canvas to write to it. Later the canvas can be rendered to a string + * suitable to be printed on the screen, using unicode Braille characters. */ + +/* This represents a very simple generic canvas in order to draw stuff. + * It's up to each LOLWUT versions to translate what they draw to the + * screen, depending on the result to accomplish. */ +typedef struct lwCanvas { + int width; + int height; + char *pixels; +} lwCanvas; + +/* Drawing functions implemented inside lolwut.c. */ +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor); +void lwFreeCanvas(lwCanvas *canvas); +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color); +int lwGetPixel(lwCanvas *canvas, int x, int y); +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color); +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color); diff --git a/src/lolwut5.c b/src/lolwut5.c new file mode 100644 index 000000000..5a9348800 --- /dev/null +++ b/src/lolwut5.c @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2018, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ---------------------------------------------------------------------------- + * + * This file implements the LOLWUT command. The command should do something + * fun and interesting, and should be replaced by a new implementation at + * each new version of Redis. + */ + +#include "server.h" +#include "lolwut.h" +#include + +/* Translate a group of 8 pixels (2x4 vertical rectangle) to the corresponding + * braille character. The byte should correspond to the pixels arranged as + * follows, where 0 is the least significant bit, and 7 the most significant + * bit: + * + * 0 3 + * 1 4 + * 2 5 + * 6 7 + * + * The corresponding utf8 encoded character is set into the three bytes + * pointed by 'output'. + */ +#include +void lwTranslatePixelsGroup(int byte, char *output) { + int code = 0x2800 + byte; + /* Convert to unicode. This is in the U0800-UFFFF range, so we need to + * emit it like this in three bytes: + * 1110xxxx 10xxxxxx 10xxxxxx. */ + output[0] = 0xE0 | (code >> 12); /* 1110-xxxx */ + output[1] = 0x80 | ((code >> 6) & 0x3F); /* 10-xxxxxx */ + output[2] = 0x80 | (code & 0x3F); /* 10-xxxxxx */ +} + +/* Schotter, the output of LOLWUT of Redis 5, is a computer graphic art piece + * generated by Georg Nees in the 60s. It explores the relationship between + * caos and order. + * + * The function creates the canvas itself, depending on the columns available + * in the output display and the number of squares per row and per column + * requested by the caller. */ +lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_col) { + /* Calculate the canvas size. */ + int canvas_width = console_cols*2; + int padding = canvas_width > 4 ? 2 : 0; + float square_side = (float)(canvas_width-padding*2) / squares_per_row; + int canvas_height = square_side * squares_per_col + padding*2; + lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height, 0); + + for (int y = 0; y < squares_per_col; y++) { + for (int x = 0; x < squares_per_row; x++) { + int sx = x * square_side + square_side/2 + padding; + int sy = y * square_side + square_side/2 + padding; + /* Rotate and translate randomly as we go down to lower + * rows. */ + float angle = 0; + if (y > 1) { + float r1 = (float)rand() / RAND_MAX / squares_per_col * y; + float r2 = (float)rand() / RAND_MAX / squares_per_col * y; + float r3 = (float)rand() / RAND_MAX / squares_per_col * y; + if (rand() % 2) r1 = -r1; + if (rand() % 2) r2 = -r2; + if (rand() % 2) r3 = -r3; + angle = r1; + sx += r2*square_side/3; + sy += r3*square_side/3; + } + lwDrawSquare(canvas,sx,sy,square_side,angle,1); + } + } + + return canvas; +} + +/* Converts the canvas to an SDS string representing the UTF8 characters to + * print to the terminal in order to obtain a graphical representaiton of the + * logical canvas. The actual returned string will require a terminal that is + * width/2 large and height/4 tall in order to hold the whole image without + * overflowing or scrolling, since each Barille character is 2x4. */ +static sds renderCanvas(lwCanvas *canvas) { + sds text = sdsempty(); + for (int y = 0; y < canvas->height; y += 4) { + for (int x = 0; x < canvas->width; x += 2) { + /* We need to emit groups of 8 bits according to a specific + * arrangement. See lwTranslatePixelsGroup() for more info. */ + int byte = 0; + if (lwGetPixel(canvas,x,y)) byte |= (1<<0); + if (lwGetPixel(canvas,x,y+1)) byte |= (1<<1); + if (lwGetPixel(canvas,x,y+2)) byte |= (1<<2); + if (lwGetPixel(canvas,x+1,y)) byte |= (1<<3); + if (lwGetPixel(canvas,x+1,y+1)) byte |= (1<<4); + if (lwGetPixel(canvas,x+1,y+2)) byte |= (1<<5); + if (lwGetPixel(canvas,x,y+3)) byte |= (1<<6); + if (lwGetPixel(canvas,x+1,y+3)) byte |= (1<<7); + char unicode[3]; + lwTranslatePixelsGroup(byte,unicode); + text = sdscatlen(text,unicode,3); + } + if (y != canvas->height-1) text = sdscatlen(text,"\n",1); + } + return text; +} + +/* The LOLWUT command: + * + * LOLWUT [terminal columns] [squares-per-row] [squares-per-col] + * + * By default the command uses 66 columns, 8 squares per row, 12 squares + * per column. + */ +void lolwut5Command(client *c) { + long cols = 66; + long squares_per_row = 8; + long squares_per_col = 12; + + /* Parse the optional arguments if any. */ + if (c->argc > 1 && + getLongFromObjectOrReply(c,c->argv[1],&cols,NULL) != C_OK) + return; + + if (c->argc > 2 && + getLongFromObjectOrReply(c,c->argv[2],&squares_per_row,NULL) != C_OK) + return; + + if (c->argc > 3 && + getLongFromObjectOrReply(c,c->argv[3],&squares_per_col,NULL) != C_OK) + return; + + /* Limits. We want LOLWUT to be always reasonably fast and cheap to execute + * so we have maximum number of columns, rows, and output resulution. */ + if (cols < 1) cols = 1; + if (cols > 1000) cols = 1000; + if (squares_per_row < 1) squares_per_row = 1; + if (squares_per_row > 200) squares_per_row = 200; + if (squares_per_col < 1) squares_per_col = 1; + if (squares_per_col > 200) squares_per_col = 200; + + /* Generate some computer art and reply. */ + lwCanvas *canvas = lwDrawSchotter(cols,squares_per_row,squares_per_col); + sds rendered = renderCanvas(canvas); + rendered = sdscat(rendered, + "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); + rendered = sdscat(rendered,REDIS_VERSION); + rendered = sdscatlen(rendered,"\n",1); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); + lwFreeCanvas(canvas); +} diff --git a/src/lolwut6.c b/src/lolwut6.c new file mode 100644 index 000000000..471bf66c8 --- /dev/null +++ b/src/lolwut6.c @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ---------------------------------------------------------------------------- + * + * This file implements the LOLWUT command. The command should do something + * fun and interesting, and should be replaced by a new implementation at + * each new version of Redis. + * + * Thanks to Michele Hiki Falcone for the original image that ispired + * the image, part of his game, Plaguemon. + * + * Thanks to the Shhh computer art collective for the help in tuning the + * output to have a better artistic effect. + */ + +#include "server.h" +#include "lolwut.h" + +/* Render the canvas using the four gray levels of the standard color + * terminal: they match very well to the grayscale display of the gameboy. */ +static sds renderCanvas(lwCanvas *canvas) { + sds text = sdsempty(); + for (int y = 0; y < canvas->height; y++) { + for (int x = 0; x < canvas->width; x++) { + int color = lwGetPixel(canvas,x,y); + char *ce; /* Color escape sequence. */ + + /* Note that we set both the foreground and background color. + * This way we are able to get a more consistent result among + * different terminals implementations. */ + switch(color) { + case 0: ce = "0;30;40m"; break; /* Black */ + case 1: ce = "0;90;100m"; break; /* Gray 1 */ + case 2: ce = "0;37;47m"; break; /* Gray 2 */ + case 3: ce = "0;97;107m"; break; /* White */ + default: ce = "0;30;40m"; break; /* Just for safety. */ + } + text = sdscatprintf(text,"\033[%s \033[0m",ce); + } + if (y != canvas->height-1) text = sdscatlen(text,"\n",1); + } + return text; +} + +/* Draw a skyscraper on the canvas, according to the parameters in the + * 'skyscraper' structure. Window colors are random and are always one + * of the two grays. */ +struct skyscraper { + int xoff; /* X offset. */ + int width; /* Pixels width. */ + int height; /* Pixels height. */ + int windows; /* Draw windows if true. */ + int color; /* Color of the skyscraper. */ +}; + +void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { + int starty = canvas->height-1; + int endy = starty - si->height + 1; + for (int y = starty; y >= endy; y--) { + for (int x = si->xoff; x < si->xoff+si->width; x++) { + /* The roof is four pixels less wide. */ + if (y == endy && (x <= si->xoff+1 || x >= si->xoff+si->width-2)) + continue; + int color = si->color; + /* Alter the color if this is a place where we want to + * draw a window. We check that we are in the inner part of the + * skyscraper, so that windows are far from the borders. */ + if (si->windows && + x > si->xoff+1 && + x < si->xoff+si->width-2 && + y > endy+1 && + y < starty-1) + { + /* Calculate the x,y position relative to the start of + * the window area. */ + int relx = x - (si->xoff+1); + int rely = y - (endy+1); + + /* Note that we want the windows to be two pixels wide + * but just one pixel tall, because terminal "pixels" + * (characters) are not square. */ + if (relx/2 % 2 && rely % 2) { + do { + color = 1 + rand() % 2; + } while (color == si->color); + /* Except we want adjacent pixels creating the same + * window to be the same color. */ + if (relx % 2) color = lwGetPixel(canvas,x-1,y); + } + } + lwDrawPixel(canvas,x,y,color); + } + } +} + +/* Generate a skyline inspired by the parallax backgrounds of 8 bit games. */ +void generateSkyline(lwCanvas *canvas) { + struct skyscraper si; + + /* First draw the background skyscraper without windows, using the + * two different grays. We use two passes to make sure that the lighter + * ones are always in the background. */ + for (int color = 2; color >= 1; color--) { + si.color = color; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 10 + rand()%9; + if (color == 2) + si.height = canvas->height/2 + rand()%canvas->height/2; + else + si.height = canvas->height/2 + rand()%canvas->height/3; + si.windows = 0; + generateSkyscraper(canvas, &si); + if (color == 2) + offset += si.width/2; + else + offset += si.width+1; + } + } + + /* Now draw the foreground skyscraper with the windows. */ + si.color = 0; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 5 + rand()%14; + if (si.width % 4) si.width += (si.width % 3); + si.height = canvas->height/3 + rand()%canvas->height/2; + si.windows = 1; + generateSkyscraper(canvas, &si); + offset += si.width+5; + } +} + +/* The LOLWUT 6 command: + * + * LOLWUT [columns] [rows] + * + * By default the command uses 80 columns, 40 squares per row + * per column. + */ +void lolwut6Command(client *c) { + long cols = 80; + long rows = 20; + + /* Parse the optional arguments if any. */ + if (c->argc > 1 && + getLongFromObjectOrReply(c,c->argv[1],&cols,NULL) != C_OK) + return; + + if (c->argc > 2 && + getLongFromObjectOrReply(c,c->argv[2],&rows,NULL) != C_OK) + return; + + /* Limits. We want LOLWUT to be always reasonably fast and cheap to execute + * so we have maximum number of columns, rows, and output resulution. */ + if (cols < 1) cols = 1; + if (cols > 1000) cols = 1000; + if (rows < 1) rows = 1; + if (rows > 1000) rows = 1000; + + /* Generate the city skyline and reply. */ + lwCanvas *canvas = lwCreateCanvas(cols,rows,3); + generateSkyline(canvas); + sds rendered = renderCanvas(canvas); + rendered = sdscat(rendered, + "\nDedicated to the 8 bit game developers of past and present.\n" + "Original 8 bit image from Plaguemon by hikikomori. Redis ver. "); + rendered = sdscat(rendered,REDIS_VERSION); + rendered = sdscatlen(rendered,"\n",1); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); + lwFreeCanvas(canvas); +} diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index e6d558b17..236c26c2b 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -3,7 +3,7 @@ GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` BUILD_ID=`uname -n`"-"`date +%s` if [ -n "$SOURCE_DATE_EPOCH" ]; then - BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u %s) + BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u +%s) fi test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ diff --git a/src/module.cpp b/src/module.cpp index dbb404082..c9cc29bfd 100644 --- a/src/module.cpp +++ b/src/module.cpp @@ -29,10 +29,12 @@ #include "server.h" #include "cluster.h" +#include "rdb.h" #include #include #include #include +#include #define REDISMODULE_CORE 1 #include "redismodule.h" @@ -43,6 +45,17 @@ * pointers that have an API the module can call with them) * -------------------------------------------------------------------------- */ +typedef struct RedisModuleInfoCtx { + struct RedisModule *module; + const char *requested_section; + sds info; /* info string we collected so far */ + int sections; /* number of sections we collected so far */ + int in_section; /* indication if we're in an active section or not */ + int in_dict_field; /* indication that we're curreintly appending to a dict */ +} RedisModuleInfoCtx; + +typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); + /* This structure represents a module inside the system. */ struct RedisModule { void *handle; /* Module dlopen() handle. */ @@ -54,6 +67,10 @@ struct RedisModule { list *usingMods; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + int in_hook; /* Hooks callback nesting level for this module (0 or 1). */ + int options; /* Module options and capabilities. */ + int blocked_clients; /* Count of RedisModuleBlockedClient in this module. */ + RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */ }; typedef struct RedisModule RedisModule; @@ -82,6 +99,7 @@ struct AutoMemEntry { #define REDISMODULE_AM_REPLY 2 #define REDISMODULE_AM_FREED 3 /* Explicitly freed by user already. */ #define REDISMODULE_AM_DICT 4 +#define REDISMODULE_AM_INFO 5 /* The pool allocator block. Redis Modules can allocate memory via this special * allocator that will automatically release it all once the callback returns. @@ -131,16 +149,23 @@ struct RedisModuleCtx { void **postponed_arrays; /* To set with RM_ReplySetArrayLength(). */ int postponed_arrays_count; /* Number of entries in postponed_arrays. */ void *blocked_privdata; /* Privdata set when unblocking a client. */ + RedisModuleString *blocked_ready_key; /* Key ready when the reply callback + gets called for clients blocked + on keys. */ /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST flag set. */ int *keys_pos; int keys_count; struct RedisModulePoolAllocBlock *pa_head; + redisOpArray saved_oparray; /* When propagating commands in a callback + we reallocate the "also propagate" op + array. Here we save the old one to + restore it later. */ }; typedef struct RedisModuleCtx RedisModuleCtx; -#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL} +#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, NULL, 0, NULL, {0}} #define REDISMODULE_CTX_MULTI_EMITTED (1<<0) #define REDISMODULE_CTX_AUTO_MEMORY (1<<1) #define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2) @@ -148,6 +173,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; #define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<4) #define REDISMODULE_CTX_THREAD_SAFE (1<<5) #define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<6) +#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<7) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -231,6 +257,8 @@ typedef struct RedisModuleBlockedClient { ::client *reply_client; /* Fake client used to accumulate replies in thread safe contexts. */ int dbid; /* Database number selected by the original client. */ + int blocked_on_keys; /* If blocked via RM_BlockClientOnKeys(). */ + int unblocked; /* Already on the moduleUnblocked list. */ } RedisModuleBlockedClient; static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER; @@ -301,6 +329,59 @@ static int s_cAcquisitionsModule = 0; static std::mutex s_mutex; static std::condition_variable s_cv; +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); + +static struct RedisModuleForkInfo { + RedisModuleForkDoneHandler done_handler; + void* done_handler_user_data; +} moduleForkInfo = {0}; + +typedef struct RedisModuleServerInfoData { + ::rax *rax; /* parsed info data. */ +} RedisModuleServerInfoData; + +/* Flags for moduleCreateArgvFromUserFormat(). */ +#define REDISMODULE_ARGV_REPLICATE (1<<0) +#define REDISMODULE_ARGV_NO_AOF (1<<1) +#define REDISMODULE_ARGV_NO_REPLICAS (1<<2) + +/* Determine whether Redis should signalModifiedKey implicitly. + * In case 'ctx' has no 'module' member (and therefore no module->options), + * we assume default behavior, that is, Redis signals. + * (see RM_GetThreadSafeContext) */ +#define SHOULD_SIGNAL_MODIFIED_KEYS(ctx) \ + ctx->module? !(ctx->module->options & REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED) : 1 + +/* Server events hooks data structures and defines: this modules API + * allow modules to subscribe to certain events in Redis, such as + * the start and end of an RDB or AOF save, the change of role in replication, + * and similar other events. */ + +typedef struct RedisModuleEventListener { + RedisModule *module; + RedisModuleEvent event; + RedisModuleEventCallback callback; +} RedisModuleEventListener; + +list *RedisModule_EventListeners; /* Global list of all the active events. */ +unsigned long long ModulesInHooks = 0; /* Total number of modules in hooks + callbacks right now. */ + +/* Data structures related to the redis module users */ + +/* This callback type is called by moduleNotifyUserChanged() every time + * a user authenticated via the module API is associated with a different + * user or gets disconnected. */ +typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata); + +/* This is the object returned by RM_CreateModuleUser(). The module API is + * able to create users, set ACLs to such users, and later authenticate + * clients using such newly created users. */ +typedef struct RedisModuleUser { + ::user *user; /* Reference to the real redis user */ +} RedisModuleUser; + + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -313,6 +394,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx); void RM_ZsetRangeStop(RedisModuleKey *kp); static void zsetKeyReset(RedisModuleKey *key); void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); +void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); /* -------------------------------------------------------------------------- * Heap allocation raw functions @@ -470,7 +552,8 @@ int moduleDelKeyIfEmpty(RedisModuleKey *key) { case OBJ_LIST: isempty = listTypeLength(o) == 0; break; case OBJ_SET: isempty = setTypeSize(o) == 0; break; case OBJ_ZSET: isempty = zsetLength(o) == 0; break; - case OBJ_HASH : isempty = hashTypeLength(o) == 0; break; + case OBJ_HASH: isempty = hashTypeLength(o) == 0; break; + case OBJ_STREAM: isempty = streamLength(o) == 0; break; default: isempty = 0; } @@ -506,8 +589,45 @@ int RM_GetApi(const char *funcname, void **targetPtrPtr) { return REDISMODULE_OK; } +/* Helper function for when a command callback is called, in order to handle + * details needed to correctly replicate commands. */ +void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { + serverAssert(GlobalLocksAcquired()); + client *c = ctx->client; + + /* We don't need to do anything here if the context was never used + * in order to propagate commands. */ + if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return; + + if (c->flags & CLIENT_LUA) return; + + /* Handle the replication of the final EXEC, since whatever a command + * emits is always wrapped around MULTI/EXEC. */ + alsoPropagate(cserver.execCommand,c->db->id,&shared.exec,1, + PROPAGATE_AOF|PROPAGATE_REPL); + + /* If this is not a module command context (but is instead a simple + * callback context), we have to handle directly the "also propagate" + * array and emit it. In a module command call this will be handled + * directly by call(). */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL) && + g_pserver->also_propagate.numops) + { + for (int j = 0; j < g_pserver->also_propagate.numops; j++) { + redisOp *rop = &g_pserver->also_propagate.ops[j]; + int target = rop->target; + if (target) + propagate(rop->cmd,rop->dbid,rop->argv,rop->argc,target); + } + redisOpArrayFree(&g_pserver->also_propagate); + /* Restore the previous oparray in case of nexted use of the API. */ + g_pserver->also_propagate = ctx->saved_oparray; + } +} + /* Free the context after the user function was called. */ void moduleFreeContext(RedisModuleCtx *ctx) { + moduleHandlePropagationAfterCommandCallback(ctx); autoMemoryCollect(ctx); poolAllocRelease(ctx); if (ctx->postponed_arrays) { @@ -523,35 +643,16 @@ void moduleFreeContext(RedisModuleCtx *ctx) { if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) freeClient(ctx->client); } -/* Helper function for when a command callback is called, in order to handle - * details needed to correctly replicate commands. */ -void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { - client *c = ctx->client; - serverAssert(GlobalLocksAcquired()); - - if (c->flags & CLIENT_LUA) return; - - /* Handle the replication of the final EXEC, since whatever a command - * emits is always wrapped around MULTI/EXEC. */ - if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) { - robj *propargv[1]; - propargv[0] = createStringObject("EXEC",4); - alsoPropagate(cserver.execCommand,c->db->id,propargv,1, - PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(propargv[0]); - } -} - /* This Redis command binds the normal Redis command invocation with commands * exported by modules. */ void RedisModuleCommandDispatcher(client *c) { RedisModuleCommandProxy *cp = (RedisModuleCommandProxy*)(unsigned long)c->cmd->getkeys_proc; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.flags |= REDISMODULE_CTX_MODULE_COMMAND_CALL; ctx.module = cp->module; ctx.client = c; cp->func(&ctx,(void**)c->argv,c->argc); - moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to @@ -643,6 +744,7 @@ int commandFlagsFromString(char *s) { else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE; else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR; else if (!strcasecmp(t,"fast")) flags |= CMD_FAST; + else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS; else if (!strcasecmp(t,"no-cluster")) flags |= CMD_MODULE_NO_CLUSTER; else break; @@ -704,6 +806,9 @@ int commandFlagsFromString(char *s) { * example, is unable to report the position of the * keys, programmatically creates key names, or any * other reason. + * * **"no-auth"**: This command can be run by an un-authenticated client. + * Normally this is used by a command that is used + * to authenticate a client. */ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { int flags = strflags ? commandFlagsFromString((char*)strflags) : 0; @@ -765,6 +870,8 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->usingMods = listCreate(); module->filters = listCreate(); module->in_call = 0; + module->in_hook = 0; + module->options = 0; ctx->module = module; } @@ -782,6 +889,25 @@ long long RM_Milliseconds(void) { return mstime(); } +/* Set flags defining capabilities or behavior bit flags. + * + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS: + * Generally, modules don't need to bother with this, as the process will just + * terminate if a read error happens, however, setting this flag would allow + * repl-diskless-load to work if enabled. + * The module should use RedisModule_IsIOError after reads, before using the + * data that was read, and in case of error, propagate it upwards, and also be + * able to release the partially populated value and all it's allocations. */ +void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { + ctx->module->options = options; +} + +/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH). */ +int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { + signalModifiedKey(ctx->client->db,keyname); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Automatic memory management for modules * -------------------------------------------------------------------------- */ @@ -857,6 +983,7 @@ void autoMemoryCollect(RedisModuleCtx *ctx) { case REDISMODULE_AM_REPLY: RM_FreeCallReply((RedisModuleCallReply*)ptr); break; case REDISMODULE_AM_KEY: RM_CloseKey((RedisModuleKey*)ptr); break; case REDISMODULE_AM_DICT: RM_FreeDict(NULL,(RedisModuleDict*)ptr); break; + case REDISMODULE_AM_INFO: RM_FreeServerInfo(NULL,(RedisModuleServerInfoData*)ptr); break; } } ctx->flags |= REDISMODULE_CTX_AUTO_MEMORY; @@ -923,6 +1050,21 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll return RM_CreateString(ctx,buf,len); } +/* Like RedisModule_CreatString(), but creates a string starting from a long + * double. + * + * The returned string must be released with RedisModule_FreeString() or by + * enabling automatic memory management. + * + * The passed context 'ctx' may be NULL if necessary, see the + * RedisModule_CreateString() documentation for more info. */ +RedisModuleString *RM_CreateStringFromLongDouble(RedisModuleCtx *ctx, long double ld, int humanfriendly) { + char buf[MAX_LONG_DOUBLE_CHARS]; + size_t len = ld2string(buf,sizeof(buf),ld, + (humanfriendly ? LD_STR_HUMAN : LD_STR_AUTO)); + return RM_CreateString(ctx,buf,len); +} + /* Like RedisModule_CreatString(), but creates a string starting from another * RedisModuleString. * @@ -1027,6 +1169,14 @@ int RM_StringToDouble(const RedisModuleString *str, double *d) { return (retval == C_OK) ? REDISMODULE_OK : REDISMODULE_ERR; } +/* Convert the string into a long double, storing it at `*ld`. + * Returns REDISMODULE_OK on success or REDISMODULE_ERR if the string is + * not a valid string representation of a double value. */ +int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) { + int retval = string2ld(szFromObj(str),sdslen(szFromObj(str)),ld); + return retval ? REDISMODULE_OK : REDISMODULE_ERR; +} + /* Compare two string objects, returning -1, 0 or 1 respectively if * a < b, a == b, a > b. Strings are compared byte by byte as two * binary blobs without any encoding care / collation attempt. */ @@ -1136,10 +1286,9 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, const char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - sds strmsg = sdsnewlen(prefix,1); - strmsg = sdscat(strmsg,msg); - strmsg = sdscatlen(strmsg,"\r\n",2); - addReplySds(c,strmsg); + addReplyProto(c,prefix,strlen(prefix)); + addReplyProto(c,msg,strlen(msg)); + addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } @@ -1196,6 +1345,27 @@ int RM_ReplyWithArray(RedisModuleCtx *ctx, long len) { return REDISMODULE_OK; } +/* Reply to the client with a null array, simply null in RESP3 + * null array in RESP2. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithNullArray(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyNullArray(c); + return REDISMODULE_OK; +} + +/* Reply to the client with an empty array. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithEmptyArray(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReply(c,shared.emptyarray); + return REDISMODULE_OK; +} + /* When RedisModule_ReplyWithArray() is used with the argument * REDISMODULE_POSTPONED_ARRAY_LEN, because we don't know beforehand the number * of items we are going to output as elements of the array, this function @@ -1274,8 +1444,28 @@ int RM_ReplyWithString(RedisModuleCtx *ctx, RedisModuleString *str) { return REDISMODULE_OK; } -/* Reply to the client with a NULL. In the RESP protocol a NULL is encoded - * as the string "$-1\r\n". +/* Reply with an empty string. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithEmptyString(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReply(c,shared.emptybulk); + return REDISMODULE_OK; +} + +/* Reply with a binary safe string, which should not be escaped or filtered + * taking in input a C buffer pointer and length. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithVerbatimString(RedisModuleCtx *ctx, const char *buf, size_t len) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyVerbatim(c, buf, len, "txt"); + return REDISMODULE_OK; +} + +/* Reply to the client with a NULL. * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithNull(RedisModuleCtx *ctx) { @@ -1312,6 +1502,21 @@ int RM_ReplyWithDouble(RedisModuleCtx *ctx, double d) { return REDISMODULE_OK; } +/* Send a string reply obtained converting the long double 'ld' into a bulk + * string. This function is basically equivalent to converting a long double + * into a string into a C buffer, and then calling the function + * RedisModule_ReplyWithStringBuffer() with the buffer and length. + * The double string uses human readable formatting (see + * `addReplyHumanLongDouble` in networking.c). + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyHumanLongDouble(c, ld); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Commands replication API * -------------------------------------------------------------------------- */ @@ -1326,9 +1531,16 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { /* If we already emitted MULTI return ASAP. */ if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) return; /* If this is a thread safe context, we do not want to wrap commands - * executed into MUTLI/EXEC, they are executed as single commands + * executed into MULTI/EXEC, they are executed as single commands * from an external client in essence. */ if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) return; + /* If this is a callback context, and not a module command execution + * context, we have to setup the op array for the "also propagate" API + * so that RM_Replicate() will work. */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) { + ctx->saved_oparray = g_pserver->also_propagate; + redisOpArrayInit(&g_pserver->also_propagate); + } execCommandPropagateMulti(ctx->client); ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED; } @@ -1350,6 +1562,24 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * * Please refer to RedisModule_Call() for more information. * + * Using the special "A" and "R" modifiers, the caller can exclude either + * the AOF or the replicas from the propagation of the specified command. + * Otherwise, by default, the command will be propagated in both channels. + * + * ## Note about calling this function from a thread safe context: + * + * Normally when you call this function from the callback implementing a + * module command, or any other callback provided by the Redis Module API, + * Redis will accumulate all the calls to this function in the context of + * the callback, and will propagate all the commands wrapped in a MULTI/EXEC + * transaction. However when calling this function from a threaded safe context + * that can live an undefined amount of time, and can be locked/unlocked in + * at will, the behavior is different: MULTI/EXEC wrapper is not emitted + * and the command specified is inserted in the AOF and replication stream + * immediately. + * + * ## Return value + * * The command returns REDISMODULE_ERR if the format specifiers are invalid * or the command name does not belong to a known command. */ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { @@ -1367,10 +1597,23 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) va_end(ap); if (argv == NULL) return REDISMODULE_ERR; - /* Replicate! */ - moduleReplicateMultiIfNeeded(ctx); - alsoPropagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + /* Select the propagation target. Usually is AOF + replicas, however + * the caller can exclude one or the other using the "A" or "R" + * modifiers. */ + int target = 0; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) target |= PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) target |= PROPAGATE_REPL; + + /* Replicate! When we are in a threaded context, we want to just insert + * the replicated command ASAP, since it is not clear when the context + * will stop being used, so accumulating stuff does not make much sense, + * nor we could easily use the alsoPropagate() API from threads. */ + if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) { + propagate(cmd,ctx->client->db->id,argv,argc,target); + } else { + moduleReplicateMultiIfNeeded(ctx); + alsoPropagate(cmd,ctx->client->db->id,argv,argc,target); + } /* Release the argv. */ for (j = 0; j < argc; j++) decrRefCount(argv[j]); @@ -1412,12 +1655,142 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) { * are guaranteed to get IDs greater than any past ID previously seen. * * Valid IDs are from 1 to 2^64-1. If 0 is returned it means there is no way - * to fetch the ID in the context the function was currently called. */ + * to fetch the ID in the context the function was currently called. + * + * After obtaining the ID, it is possible to check if the command execution + * is actually happening in the context of AOF loading, using this macro: + * + * if (RedisModule_IsAOFClient(RedisModule_GetClientId(ctx)) { + * // Handle it differently. + * } + */ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { if (ctx->client == NULL) return 0; return ctx->client->id; } +/* This is an helper for RM_GetClientInfoById() and other functions: given + * a client, it populates the client info structure with the appropriate + * fields depending on the version provided. If the version is not valid + * then REDISMODULE_ERR is returned. Otherwise the function returns + * REDISMODULE_OK and the structure pointed by 'ci' gets populated. */ + +int modulePopulateClientInfoStructure(void *ci, client *client, int structver) { + if (structver != 1) return REDISMODULE_ERR; + + RedisModuleClientInfoV1 *ci1 = (RedisModuleClientInfoV1*)ci; + memset(ci1,0,sizeof(*ci1)); + ci1->version = structver; + if (client->flags & CLIENT_MULTI) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_MULTI; + if (client->flags & CLIENT_PUBSUB) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_PUBSUB; + if (client->flags & CLIENT_UNIX_SOCKET) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET; + if (client->flags & CLIENT_TRACKING) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_TRACKING; + if (client->flags & CLIENT_BLOCKED) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_BLOCKED; + + int port; + connPeerToString(client->conn,ci1->addr,sizeof(ci1->addr),&port); + ci1->port = port; + ci1->db = client->db->id; + ci1->id = client->id; + return REDISMODULE_OK; +} + +/* This is an helper for moduleFireServerEvent() and other functions: + * It populates the replication info structure with the appropriate + * fields depending on the version provided. If the version is not valid + * then REDISMODULE_ERR is returned. Otherwise the function returns + * REDISMODULE_OK and the structure pointed by 'ri' gets populated. */ +int modulePopulateReplicationInfoStructure(void *ri, int structver) { + if (structver != 1) return REDISMODULE_ERR; + + RedisModuleReplicationInfoV1 *ri1 = (RedisModuleReplicationInfoV1*)ri; + memset(ri1,0,sizeof(*ri1)); + ri1->version = structver; + ri1->master = listLength(g_pserver->masters) == 0; + if (ri1->master) + { + redisMaster *mi = (redisMaster*)listFirst(g_pserver->masters); + ri1->masterhost = (char*)(mi->masterhost? mi->masterhost: ""); + ri1->masterport = mi->masterport; + } + else + { + ri1->masterhost = nullptr; + ri1->masterport = -1; + } + ri1->repl1_offset = g_pserver->master_repl_offset; + ri1->repl2_offset = g_pserver->second_replid_offset; + ri1->replid1 = g_pserver->replid; + ri1->replid2 = g_pserver->replid2; + return REDISMODULE_OK; +} + +/* Return information about the client with the specified ID (that was + * previously obtained via the RedisModule_GetClientId() API). If the + * client exists, REDISMODULE_OK is returned, otherwise REDISMODULE_ERR + * is returned. + * + * When the client exist and the `ci` pointer is not NULL, but points to + * a structure of type RedisModuleClientInfo, previously initialized with + * the correct REDISMODULE_CLIENTINFO_INITIALIZER, the structure is populated + * with the following fields: + * + * uint64_t flags; // REDISMODULE_CLIENTINFO_FLAG_* + * uint64_t id; // Client ID + * char addr[46]; // IPv4 or IPv6 address. + * uint16_t port; // TCP port. + * uint16_t db; // Selected DB. + * + * Note: the client ID is useless in the context of this call, since we + * already know, however the same structure could be used in other + * contexts where we don't know the client ID, yet the same structure + * is returned. + * + * With flags having the following meaning: + * + * REDISMODULE_CLIENTINFO_FLAG_SSL Client using SSL connection. + * REDISMODULE_CLIENTINFO_FLAG_PUBSUB Client in Pub/Sub mode. + * REDISMODULE_CLIENTINFO_FLAG_BLOCKED Client blocked in command. + * REDISMODULE_CLIENTINFO_FLAG_TRACKING Client with keys tracking on. + * REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET Client using unix domain socket. + * REDISMODULE_CLIENTINFO_FLAG_MULTI Client in MULTI state. + * + * However passing NULL is a way to just check if the client exists in case + * we are not interested in any additional information. + * + * This is the correct usage when we want the client info structure + * returned: + * + * RedisModuleClientInfo ci = REDISMODULE_CLIENTINFO_INITIALIZER; + * int retval = RedisModule_GetClientInfoById(&ci,client_id); + * if (retval == REDISMODULE_OK) { + * printf("Address: %s\n", ci.addr); + * } + */ +int RM_GetClientInfoById(void *ci, uint64_t id) { + client *client = lookupClientByID(id); + if (client == NULL) return REDISMODULE_ERR; + if (ci == NULL) return REDISMODULE_OK; + + /* Fill the info structure if passed. */ + uint64_t structver = ((uint64_t*)ci)[0]; + return modulePopulateClientInfoStructure(ci,client,structver); +} + +/* Publish a message to subscribers (see PUBLISH command). */ +int RM_PublishMessage(RedisModuleCtx *ctx, RedisModuleString *channel, RedisModuleString *message) { + UNUSED(ctx); + int receivers = pubsubPublishMessage(channel, message); + if (g_pserver->cluster_enabled) + clusterPropagatePublish(channel, message); + return receivers; +} + /* Return the currently selected DB. */ int RM_GetSelectedDb(RedisModuleCtx *ctx) { return ctx->client->db->id; @@ -1459,6 +1832,23 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { * * * REDISMODULE_CTX_FLAGS_OOM_WARNING: Less than 25% of memory remains before * reaching the maxmemory level. + * + * * REDISMODULE_CTX_FLAGS_LOADING: Server is loading RDB/AOF + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE: No active link with the master. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING: The replica is trying to + * connect with the master. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING: Master -> Replica RDB + * transfer is in progress. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE: The replica has an active link + * with its master. This is the + * contrary of STALE state. + * + * * REDISMODULE_CTX_FLAGS_ACTIVE_CHILD: There is currently some background + * process active (RDB, AUX or module). */ int RM_GetContextFlags(RedisModuleCtx *ctx) { @@ -1474,6 +1864,12 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { flags |= REDISMODULE_CTX_FLAGS_REPLICATED; } + /* For DIRTY flags, we need the blocked client if used */ + client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; + if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) { + flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY; + } + if (g_pserver->cluster_enabled) flags |= REDISMODULE_CTX_FLAGS_CLUSTER; @@ -1501,6 +1897,21 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { flags |= REDISMODULE_CTX_FLAGS_SLAVE; if (g_pserver->repl_slave_ro) flags |= REDISMODULE_CTX_FLAGS_READONLY; + + /* Replica state flags. */ + redisMaster *mi = (redisMaster*)listFirst(g_pserver->masters); + if (mi->repl_state == REPL_STATE_CONNECT || + mi->repl_state == REPL_STATE_CONNECTING) + { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING; + } else if (mi->repl_state == REPL_STATE_TRANSFER) { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING; + } else if (mi->repl_state == REPL_STATE_CONNECTED) { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE; + } + + if (mi->repl_state != REPL_STATE_CONNECTED) + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE; } /* OOM flag. */ @@ -1509,9 +1920,36 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { if (retval == C_ERR) flags |= REDISMODULE_CTX_FLAGS_OOM; if (level > 0.75) flags |= REDISMODULE_CTX_FLAGS_OOM_WARNING; + /* Presence of children processes. */ + if (hasActiveChildProcess()) flags |= REDISMODULE_CTX_FLAGS_ACTIVE_CHILD; + return flags; } +/* Returns true if some client sent the CLIENT PAUSE command to the server or + * if Redis Cluster is doing a manual failover, and paused tue clients. + * This is needed when we have a master with replicas, and want to write, + * without adding further data to the replication channel, that the replicas + * replication offset, match the one of the master. When this happens, it is + * safe to failover the master without data loss. + * + * However modules may generate traffic by calling RedisModule_Call() with + * the "!" flag, or by calling RedisModule_Replicate(), in a context outside + * commands execution, for instance in timeout callbacks, threads safe + * contexts, and so forth. When modules will generate too much traffic, it + * will be hard for the master and replicas offset to match, because there + * is more data to send in the replication channel. + * + * So modules may want to try to avoid very heavy background work that has + * the effect of creating data to the replication channel, when this function + * returns true. This is mostly useful for modules that have background + * garbage collection tasks, or that do writes and replicate such writes + * periodically in timer callbacks or other periodic callbacks. + */ +int RM_AvoidReplicaTraffic() { + return clientsArePaused(); +} + /* Change the currently selected DB. Returns an error if the id * is out of range. * @@ -1527,6 +1965,18 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { return (retval == C_OK) ? REDISMODULE_OK : REDISMODULE_ERR; } +/* Initialize a RedisModuleKey struct */ +static void moduleInitKey(RedisModuleKey *kp, RedisModuleCtx *ctx, robj *keyname, robj *value, int mode){ + kp->ctx = ctx; + kp->db = ctx->client->db; + kp->key = keyname; + incrRefCount(keyname); + kp->value = value; + kp->iter = NULL; + kp->mode = mode; + zsetKeyReset(kp); +} + /* Return an handle representing a Redis key, so that it is possible * to call other APIs with the key handle as argument to perform * operations on the key. @@ -1544,37 +1994,38 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { RedisModuleKey *kp; robj *value; + int flags = mode & REDISMODULE_OPEN_KEY_NOTOUCH? LOOKUP_NOTOUCH: 0; if (mode & REDISMODULE_WRITE) { - value = lookupKeyWrite(ctx->client->db,keyname); + value = lookupKeyWriteWithFlags(ctx->client->db,keyname, flags); } else { - value = lookupKeyRead(ctx->client->db,keyname).unsafe_robjcast(); + value = lookupKeyReadWithFlags(ctx->client->db,keyname, flags).unsafe_robjcast(); if (value == NULL) { return NULL; } } /* Setup the key handle. */ - kp = (RedisModuleKey*)zmalloc(sizeof(*kp), MALLOC_LOCAL); - kp->ctx = ctx; - kp->db = ctx->client->db; - kp->key = keyname; - incrRefCount(keyname); - kp->value = value; - kp->iter = NULL; - kp->mode = mode; - zsetKeyReset(kp); + kp = (RedisModuleKey*)zmalloc(sizeof(*kp)); + moduleInitKey(kp, ctx, keyname, value, mode); autoMemoryAdd(ctx,REDISMODULE_AM_KEY,kp); return (void*)kp; } +/* Destroy a RedisModuleKey struct (freeing is the responsibility of the caller). */ +static void moduleCloseKey(RedisModuleKey *key) { + int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); + if ((key->mode & REDISMODULE_WRITE) && signal) + signalModifiedKey(key->db,key->key); + /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */ + RM_ZsetRangeStop(key); + decrRefCount(key->key); +} + /* Close a key handle. */ void RM_CloseKey(RedisModuleKey *key) { if (key == NULL) return; - if (key->mode & REDISMODULE_WRITE) signalModifiedKey(key->db,key->key); - /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */ - RM_ZsetRangeStop(key); - decrRefCount(key->key); + moduleCloseKey(key); autoMemoryFreed(key->ctx,REDISMODULE_AM_KEY,key); zfree(key); } @@ -1592,6 +2043,7 @@ int RM_KeyType(RedisModuleKey *key) { case OBJ_ZSET: return REDISMODULE_KEYTYPE_ZSET; case OBJ_HASH: return REDISMODULE_KEYTYPE_HASH; case OBJ_MODULE: return REDISMODULE_KEYTYPE_MODULE; + case OBJ_STREAM: return REDISMODULE_KEYTYPE_STREAM; default: return 0; } } @@ -1609,6 +2061,7 @@ size_t RM_ValueLength(RedisModuleKey *key) { case OBJ_SET: return setTypeSize(key->value); case OBJ_ZSET: return zsetLength(key->value); case OBJ_HASH: return hashTypeLength(key->value); + case OBJ_STREAM: return streamLength(key->value); default: return 0; } } @@ -1626,7 +2079,7 @@ int RM_DeleteKey(RedisModuleKey *key) { return REDISMODULE_OK; } -/* If the key is open for writing, unlink it (that is delete it in a +/* If the key is open for writing, unlink it (that is delete it in a * non-blocking way, not reclaiming memory immediately) and setup the key to * accept new writes as an empty key (that will be created on demand). * On success REDISMODULE_OK is returned. If the key is not open for @@ -1675,6 +2128,28 @@ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { return REDISMODULE_OK; } +/* Performs similar operation to FLUSHALL, and optionally start a new AOF file (if enabled) + * If restart_aof is true, you must make sure the command that triggered this call is not + * propagated to the AOF file. + * When async is set to true, db contents will be freed by a background thread. */ +void RM_ResetDataset(int restart_aof, int async) { + if (restart_aof && g_pserver->aof_state != AOF_OFF) stopAppendOnly(); + flushAllDataAndResetRDB(async? EMPTYDB_ASYNC: EMPTYDB_NO_FLAGS); + if (g_pserver->aof_enabled && restart_aof) restartAOFAfterSYNC(); +} + +/* Returns the number of keys in the current db. */ +unsigned long long RM_DbSize(RedisModuleCtx *ctx) { + return dictSize(ctx->client->db->pdict); +} + +/* Returns a name of a random key, or NULL if current db is empty. */ +RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) { + robj *key = dbRandomKey(ctx->client->db); + autoMemoryAdd(ctx,REDISMODULE_AM_STRING,key); + return key; +} + /* -------------------------------------------------------------------------- * Key API for String type * -------------------------------------------------------------------------- */ @@ -2403,7 +2878,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { * * REDISMODULE_HASH_EXISTS: instead of setting the value of the field * expecting a RedisModuleString pointer to pointer, the function just - * reports if the field esists or not and expects an integer pointer + * reports if the field exists or not and expects an integer pointer * as the second element of each pair. * * Example of REDISMODULE_HASH_CFIELD: @@ -2692,12 +3167,11 @@ RedisModuleString *RM_CreateStringFromCallReply(RedisModuleCallReply *reply) { * to special modifiers in "fmt". For now only one exists: * * "!" -> REDISMODULE_ARGV_REPLICATE + * "A" -> REDISMODULE_ARGV_NO_AOF + * "R" -> REDISMODULE_ARGV_NO_REPLICAS * * On error (format specifier error) NULL is returned and nothing is * allocated. On success the argument vector is returned. */ - -#define REDISMODULE_ARGV_REPLICATE (1<<0) - robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap) { int argc = 0, argv_size, j; robj **argv = NULL; @@ -2726,7 +3200,7 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int size_t len = va_arg(ap,size_t); argv[argc++] = createStringObject(buf,len); } else if (*p == 'l') { - long ll = va_arg(ap,long long); + long long ll = va_arg(ap,long long); argv[argc++] = createObject(OBJ_STRING,sdsfromlonglong(ll)); } else if (*p == 'v') { /* A vector of strings */ @@ -2746,6 +3220,10 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int } } else if (*p == '!') { if (flags) (*flags) |= REDISMODULE_ARGV_REPLICATE; + } else if (*p == 'A') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_AOF; + } else if (*p == 'R') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_REPLICAS; } else { goto fmterr; } @@ -2765,8 +3243,16 @@ fmterr: * On success a RedisModuleCallReply object is returned, otherwise * NULL is returned and errno is set to the following values: * - * EINVAL: command non existing, wrong arity, wrong format specifier. - * EPERM: operation in Cluster instance with key in non local slot. */ + * EBADF: wrong format specifier. + * EINVAL: wrong command arity. + * ENOENT: command does not exist. + * EPERM: operation in Cluster instance with key in non local slot. + * EROFS: operation in Cluster instance when a write command is sent + * in a readonly state. + * ENETDOWN: operation in Cluster instance when cluster is down. + * + * This API is documented here: https://redis.io/topics/modules-intro + */ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { struct redisCommand *cmd; client *c = NULL; @@ -2780,7 +3266,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* Create the client and dispatch the command. */ va_start(ap, fmt); - c = createClient(-1, IDX_EVENT_LOOP_MAIN); + c = createClient(NULL, IDX_EVENT_LOOP_MAIN); c->puser = NULL; /* Root user. */ argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&flags,ap); replicate = flags & REDISMODULE_ARGV_REPLICATE; @@ -2795,7 +3281,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* We handle the above format error only when the client is setup so that * we can free it normally. */ - if (argv == NULL) goto cleanup; + if (argv == NULL) { + errno = EBADF; + goto cleanup; + } /* Call command filters */ moduleCallCommandFilters(c); @@ -2805,7 +3294,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch */ cmd = lookupCommand(szFromObj(c->argv[0])); if (!cmd) { - errno = EINVAL; + errno = ENOENT; goto cleanup; } c->cmd = c->lastcmd = cmd; @@ -2820,13 +3309,20 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch * trying to access non-local keys, with the exception of commands * received from our master. */ if (g_pserver->cluster_enabled && !(ctx->client->flags & CLIENT_MASTER)) { + int error_code; /* Duplicate relevant flags in the module client. */ c->flags &= ~(CLIENT_READONLY|CLIENT_ASKING); c->flags |= ctx->client->flags & (CLIENT_READONLY|CLIENT_ASKING); - if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,NULL) != + if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) != g_pserver->cluster->myself) { - errno = EPERM; + if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { + errno = EROFS; + } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { + errno = ENETDOWN; + } else { + errno = EPERM; + } goto cleanup; } } @@ -2836,11 +3332,18 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch * a Lua script in the context of AOF and slaves. */ if (replicate) moduleReplicateMultiIfNeeded(ctx); + if (ctx->client->flags & CLIENT_MULTI || + ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) { + c->flags |= CLIENT_MULTI; + } + /* Run the command */ call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; if (replicate) { - call_flags |= CMD_CALL_PROPAGATE_AOF; - call_flags |= CMD_CALL_PROPAGATE_REPL; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) + call_flags |= CMD_CALL_PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) + call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c,call_flags); @@ -3095,6 +3598,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, moduleTypeMemUsageFunc mem_usage; moduleTypeDigestFunc digest; moduleTypeFreeFunc free; + struct { + moduleTypeAuxLoadFunc aux_load; + moduleTypeAuxSaveFunc aux_save; + int aux_save_triggers; + } v2; } *tms = (struct typemethods*) typemethods_ptr; moduleType *mt = (moduleType*)zcalloc(sizeof(*mt), MALLOC_LOCAL); @@ -3106,6 +3614,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, mt->mem_usage = tms->mem_usage; mt->digest = tms->digest; mt->free = tms->free; + if (tms->version >= 2) { + mt->aux_load = tms->v2.aux_load; + mt->aux_save = tms->v2.aux_save; + mt->aux_save_triggers = tms->v2.aux_save_triggers; + } memcpy(mt->name,name,sizeof(mt->name)); listAddNodeTail(ctx->module->types,mt); return mt; @@ -3156,9 +3669,14 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * RDB loading and saving functions * -------------------------------------------------------------------------- */ -/* Called when there is a load error in the context of a module. This cannot - * be recovered like for the built-in types. */ +/* Called when there is a load error in the context of a module. On some + * modules this cannot be recovered, but if the module declared capability + * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { + if (io->type->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { + io->error = 1; + return; + } serverLog(LL_WARNING, "Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " @@ -3169,6 +3687,33 @@ void moduleRDBLoadError(RedisModuleIO *io) { exit(1); } +/* Returns 0 if there's at least one registered data type that did not declare + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS, in which case diskless loading should + * be avoided since it could cause data loss. */ +int moduleAllDatatypesHandleErrors() { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = (RedisModule*)dictGetVal(de); + if (listLength(module->types) && + !(module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS)) + { + dictReleaseIterator(di); + return 0; + } + } + dictReleaseIterator(di); + return 1; +} + +/* Returns true if any previous IO API failed. + * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetModuleOptions first. */ +int RM_IsIOError(RedisModuleIO *io) { + return io->error; +} + /* Save an unsigned 64 bit value into the RDB file. This function should only * be called in the context of the rdb_save method of modules implementing new * data types. */ @@ -3193,6 +3738,7 @@ saveerr: * new data types. */ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { int retval; + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; @@ -3204,7 +3750,7 @@ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* Like RedisModule_SaveUnsigned() but for signed 64 bit values. */ @@ -3264,6 +3810,7 @@ saveerr: /* Implements RM_LoadString() and RM_LoadStringBuffer() */ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { void *s = nullptr; + if (io->error) return NULL; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; @@ -3275,7 +3822,7 @@ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { loaderr: moduleRDBLoadError(io); - return NULL; /* Never reached. */ + return NULL; } /* In the context of the rdb_load method of a module data type, loads a string @@ -3296,7 +3843,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) { * RedisModule_Realloc() or RedisModule_Free(). * * The size of the string is stored at '*lenptr' if not NULL. - * The returned string is not automatically NULL termianted, it is loaded + * The returned string is not automatically NULL terminated, it is loaded * exactly as it was stored inisde the RDB file. */ char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) { return (char*)moduleLoadString(io,1,lenptr); @@ -3325,6 +3872,7 @@ saveerr: * double value saved by RedisModule_SaveDouble(). */ double RM_LoadDouble(RedisModuleIO *io) { int retval; + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; @@ -3336,7 +3884,7 @@ double RM_LoadDouble(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* In the context of the rdb_save method of a module data type, saves a float @@ -3362,6 +3910,7 @@ saveerr: * float value saved by RedisModule_SaveFloat(). */ float RM_LoadFloat(RedisModuleIO *io) { int retval = C_ERR; + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; @@ -3373,7 +3922,62 @@ float RM_LoadFloat(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; +} + +/* In the context of the rdb_save method of a module data type, saves a long double + * value to the RDB file. The double can be a valid number, a NaN or infinity. + * It is possible to load back the value with RedisModule_LoadLongDouble(). */ +void RM_SaveLongDouble(RedisModuleIO *io, long double value) { + if (io->error) return; + char buf[MAX_LONG_DOUBLE_CHARS]; + /* Long double has different number of bits in different platforms, so we + * save it as a string type. */ + size_t len = ld2string(buf,sizeof(buf),value,LD_STR_HEX); + RM_SaveStringBuffer(io,buf,len+1); /* len+1 for '\0' */ +} + +/* In the context of the rdb_save method of a module data type, loads back the + * long double value saved by RedisModule_SaveLongDouble(). */ +long double RM_LoadLongDouble(RedisModuleIO *io) { + if (io->error) return 0; + long double value; + size_t len; + char* str = RM_LoadStringBuffer(io,&len); + if (!str) return 0; + string2ld(str,len,&value); + RM_Free(str); + return value; +} + +/* Iterate over modules, and trigger rdb aux saving for the ones modules types + * who asked for it. */ +ssize_t rdbSaveModulesAux(rio *rdb, int when) { + size_t total_written = 0; + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = (RedisModule*)dictGetVal(de); + listIter li; + listNode *ln; + + listRewind(module->types,&li); + while((ln = listNext(&li))) { + moduleType *mt = (moduleType*)ln->value; + if (!mt->aux_save || !(mt->aux_save_triggers & when)) + continue; + ssize_t ret = rdbSaveSingleModuleAux(rdb, when, mt); + if (ret==-1) { + dictReleaseIterator(di); + return -1; + } + total_written += ret; + } + } + + dictReleaseIterator(di); + return total_written; } /* -------------------------------------------------------------------------- @@ -3436,6 +4040,69 @@ void RM_DigestEndSequence(RedisModuleDigest *md) { memset(md->o,0,sizeof(md->o)); } +/* Decode a serialized representation of a module data type 'mt' from string + * 'str' and return a newly allocated value, or NULL if decoding failed. + * + * This call basically reuses the 'rdb_load' callback which module data types + * implement in order to allow a module to arbitrarily serialize/de-serialize + * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented. + * + * Modules should generally use the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag and + * make sure the de-serialization code properly checks and handles IO errors + * (freeing allocated buffers and returning a NULL). + * + * If this is NOT done, Redis will handle corrupted (or just truncated) serialized + * data by producing an error message and terminating the process. + */ + +void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType *mt) { + rio payload; + RedisModuleIO io; + void *ret; + + rioInitWithBuffer(&payload, szFromObj(str)); + moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); + + /* All RM_Save*() calls always write a version 2 compatible format, so we + * need to make sure we read the same. + */ + io.ver = 2; + ret = mt->rdb_load(&io,0); + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + return ret; +} + +/* Encode a module data type 'mt' value 'data' into serialized form, and return it + * as a newly allocated RedisModuleString. + * + * This call basically reuses the 'rdb_save' callback which module data types + * implement in order to allow a module to arbitrarily serialize/de-serialize + * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented. + */ + +RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, const moduleType *mt) { + rio payload; + RedisModuleIO io; + + rioInitWithBuffer(&payload,sdsempty()); + moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); + mt->rdb_save(&io,data); + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + if (io.error) { + return NULL; + } else { + robj *str = createObject(OBJ_STRING,payload.io.buffer.ptr); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_STRING,str); + return str; + } +} + /* -------------------------------------------------------------------------- * AOF API for modules data types * -------------------------------------------------------------------------- */ @@ -3513,6 +4180,11 @@ const RedisModuleString *RM_GetKeyNameFromIO(RedisModuleIO *io) { return io->key; } +/* Returns a RedisModuleString with the name of the key from RedisModuleKey */ +const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) { + return key ? key->key : NULL; +} + /* -------------------------------------------------------------------------- * Logging * -------------------------------------------------------------------------- */ @@ -3536,7 +4208,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li if (level < cserver.verbosity) return; - name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name); + name_len = snprintf(msg, sizeof(msg),"<%s> ", module? module->name: "module"); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); serverLogRaw(level,msg); } @@ -3554,13 +4226,15 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li * There is a fixed limit to the length of the log line this function is able * to emit, this limit is not specified but is guaranteed to be more than * a few lines of text. + * + * The ctx argument may be NULL if cannot be provided in the context of the + * caller for instance threads or callbacks, in which case a generic "module" + * will be used instead of the module name. */ void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { - if (!ctx->module) return; /* Can only log if module is initialized */ - va_list ap; va_start(ap, fmt); - RM_LogRaw(ctx->module,levelstr,fmt,ap); + RM_LogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap); va_end(ap); } @@ -3576,6 +4250,23 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ... va_end(ap); } +/* Redis-like assert function. + * + * A failed assertion will shut down the server and produce logging information + * that looks identical to information generated by Redis itself. + */ +void RM__Assert(const char *estr, const char *file, int line) { + _serverAssert(estr, file, line); +} + +/* Allows adding event to the latency monitor to be observed by the LATENCY + * command. The call is skipped if the latency is smaller than the configured + * latency-monitor-threshold. */ +void RM_LatencyAddSample(const char *event, mstime_t latency) { + if (latency >= g_pserver->latency_monitor_threshold) + latencyAddSample(event, latency); +} + /* -------------------------------------------------------------------------- * Blocking clients from modules * -------------------------------------------------------------------------- */ @@ -3606,7 +4297,10 @@ void moduleBlockedClientPipeReadable(aeEventLoop *el, int fd, void *privdata, in void unblockClientFromModule(client *c) { RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle; - /* Call the disconnection callback if any. */ + /* Call the disconnection callback if any. Note that + * bc->disconnect_callback is set to NULL if the client gets disconnected + * by the module itself or because of a timeout, so the callback will NOT + * get called if this is not an actual disconnection event. */ if (bc->disconnect_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.blocked_privdata = bc->privdata; @@ -3624,6 +4318,94 @@ void unblockClientFromModule(client *c) { resetClient(c); } +/* Block a client in the context of a module: this function implements both + * RM_BlockClient() and RM_BlockClientOnKeys() depending on the fact the + * keys are passed or not. + * + * When not blocking for keys, the keys, numkeys, and privdata parameters are + * not needed. The privdata in that case must be NULL, since later is + * RM_UnblockClient() that will provide some private data that the reply + * callback will receive. + * + * Instead when blocking for keys, normally RM_UnblockClient() will not be + * called (because the client will unblock when the key is modified), so + * 'privdata' should be provided in that case, so that once the client is + * unlocked and the reply callback is called, it will receive its associated + * private data. + * + * Even when blocking on keys, RM_UnblockClient() can be called however, but + * in that case the privdata argument is disregarded, because we pass the + * reply callback the privdata that is set here while blocking. + */ +RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { + client *c = ctx->client; + int islua = c->flags & CLIENT_LUA; + int ismulti = c->flags & CLIENT_MULTI; + + c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient)); + RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle; + ctx->module->blocked_clients++; + + /* We need to handle the invalid operation of calling modules blocking + * commands from Lua or MULTI. We actually create an already aborted + * (client set to NULL) blocked client handle, and actually reply with + * an error. */ + mstime_t timeout = timeout_ms ? (mstime()+timeout_ms) : 0; + bc->client = (islua || ismulti) ? NULL : c; + bc->module = ctx->module; + bc->reply_callback = reply_callback; + bc->timeout_callback = timeout_callback; + bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */ + bc->free_privdata = free_privdata; + bc->privdata = privdata; + bc->reply_client = createClient(NULL, IDX_EVENT_LOOP_MAIN); + bc->reply_client->flags |= CLIENT_MODULE; + bc->dbid = c->db->id; + bc->blocked_on_keys = keys != NULL; + bc->unblocked = 0; + c->bpop.timeout = timeout; + + if (islua || ismulti) { + c->bpop.module_blocked_handle = NULL; + addReplyError(c, islua ? + "Blocking module command called from Lua script" : + "Blocking module command called from transaction"); + } else { + if (keys) { + blockForKeys(c,BLOCKED_MODULE,keys,numkeys,timeout,NULL,NULL); + } else { + blockClient(c,BLOCKED_MODULE); + } + } + return bc; +} + +/* This function is called from module.c in order to check if a module + * blocked for BLOCKED_MODULE and subtype 'on keys' (bc->blocked_on_keys true) + * can really be unblocked, since the module was able to serve the client. + * If the callback returns REDISMODULE_OK, then the client can be unblocked, + * otherwise the client remains blocked and we'll retry again when one of + * the keys it blocked for becomes "ready" again. */ +int moduleTryServeClientBlockedOnKey(client *c, robj *key) { + int served = 0; + RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle; + /* Protect against re-processing: don't serve clients that are already + * in the unblocking list for any reason (including RM_UnblockClient() + * explicit call). */ + if (bc->unblocked) return REDISMODULE_ERR; + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; + ctx.blocked_ready_key = key; + ctx.blocked_privdata = bc->privdata; + ctx.module = bc->module; + ctx.client = bc->client; + ctx.blocked_client = bc; + if (bc->reply_callback(&ctx,(void**)c->argv,c->argc) == REDISMODULE_OK) + served = 1; + moduleFreeContext(&ctx); + return served; +} + /* Block a client in the context of a blocking command, returning an handle * which will be used, later, in order to unblock the client with a call to * RedisModule_UnblockClient(). The arguments specify callback functions @@ -3641,38 +4423,96 @@ void unblockClientFromModule(client *c) { * by RedisModule_UnblockClient() call. */ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { - client *c = ctx->client; - int islua = c->flags & CLIENT_LUA; - int ismulti = c->flags & CLIENT_MULTI; + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL); +} - c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient), MALLOC_LOCAL); - RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle; +/* This call is similar to RedisModule_BlockClient(), however in this case we + * don't just block the client, but also ask Redis to unblock it automatically + * once certain keys become "ready", that is, contain more data. + * + * Basically this is similar to what a typical Redis command usually does, + * like BLPOP or ZPOPMAX: the client blocks if it cannot be served ASAP, + * and later when the key receives new data (a list push for instance), the + * client is unblocked and served. + * + * However in the case of this module API, when the client is unblocked? + * + * 1. If you block ok a key of a type that has blocking operations associated, + * like a list, a sorted set, a stream, and so forth, the client may be + * unblocked once the relevant key is targeted by an operation that normally + * unblocks the native blocking operations for that type. So if we block + * on a list key, an RPUSH command may unblock our client and so forth. + * 2. If you are implementing your native data type, or if you want to add new + * unblocking conditions in addition to "1", you can call the modules API + * RedisModule_SignalKeyAsReady(). + * + * Anyway we can't be sure if the client should be unblocked just because the + * key is signaled as ready: for instance a successive operation may change the + * key, or a client in queue before this one can be served, modifying the key + * as well and making it empty again. So when a client is blocked with + * RedisModule_BlockClientOnKeys() the reply callback is not called after + * RM_UnblockCLient() is called, but every time a key is signaled as ready: + * if the reply callback can serve the client, it returns REDISMODULE_OK + * and the client is unblocked, otherwise it will return REDISMODULE_ERR + * and we'll try again later. + * + * The reply callback can access the key that was signaled as ready by + * calling the API RedisModule_GetBlockedClientReadyKey(), that returns + * just the string name of the key as a RedisModuleString object. + * + * Thanks to this system we can setup complex blocking scenarios, like + * unblocking a client only if a list contains at least 5 items or other + * more fancy logics. + * + * Note that another difference with RedisModule_BlockClient(), is that here + * we pass the private data directly when blocking the client: it will + * be accessible later in the reply callback. Normally when blocking with + * RedisModule_BlockClient() the private data to reply to the client is + * passed when calling RedisModule_UnblockClient() but here the unblocking + * is performed by Redis itself, so we need to have some private data before + * hand. The private data is used to store any information about the specific + * unblocking operation that you are implementing. Such information will be + * freed using the free_privdata callback provided by the user. + * + * However the reply callback will be able to access the argument vector of + * the command, so the private data is often not needed. */ +RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata); +} - /* We need to handle the invalid operation of calling modules blocking - * commands from Lua or MULTI. We actually create an already aborted - * (client set to NULL) blocked client handle, and actually reply with - * an error. */ - bc->client = (islua || ismulti) ? NULL : c; - bc->module = ctx->module; - bc->reply_callback = reply_callback; - bc->timeout_callback = timeout_callback; - bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */ - bc->free_privdata = free_privdata; - bc->privdata = NULL; - bc->reply_client = createClient(-1, IDX_EVENT_LOOP_MAIN); - bc->reply_client->flags |= CLIENT_MODULE; - bc->dbid = c->db->id; - c->bpop.timeout = timeout_ms ? (mstime()+timeout_ms) : 0; +/* This function is used in order to potentially unblock a client blocked + * on keys with RedisModule_BlockClientOnKeys(). When this function is called, + * all the clients blocked for this key will get their reply callback called, + * and if the callback returns REDISMODULE_OK the client will be unblocked. */ +void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) { + signalKeyAsReady(ctx->client->db, key); +} - if (islua || ismulti) { - c->bpop.module_blocked_handle = NULL; - addReplyError(c, islua ? - "Blocking module command called from Lua script" : - "Blocking module command called from transaction"); - } else { - blockClient(c,BLOCKED_MODULE); +/* Implements RM_UnblockClient() and moduleUnblockClient(). */ +int moduleUnblockClientByHandle(RedisModuleBlockedClient *bc, void *privdata) { + pthread_mutex_lock(&moduleUnblockedClientsMutex); + if (!bc->blocked_on_keys) bc->privdata = privdata; + bc->unblocked = 1; + listAddNodeTail(moduleUnblockedClients,bc); + if (write(serverTL->module_blocked_pipe[1],"A",1) != 1) { + /* Ignore the error, this is best-effort. */ } - return bc; + pthread_mutex_unlock(&moduleUnblockedClientsMutex); + return REDISMODULE_OK; +} + +/* This API is used by the Redis core to unblock a client that was blocked + * by a module. */ +void moduleUnblockClient(client *c) { + RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle; + moduleUnblockClientByHandle(bc,NULL); +} + +/* Return true if the client 'c' was blocked by a module using + * RM_BlockClientOnKeys(). */ +int moduleClientIsBlockedOnKeys(client *c) { + RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle; + return bc->blocked_on_keys; } /* Unblock a client blocked by `RedisModule_BlockedClient`. This will trigger @@ -3685,28 +4525,25 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * needs to be passed to the client, included but not limited some slow * to compute reply or some reply obtained via networking. * - * Note: this function can be called from threads spawned by the module. */ + * Note 1: this function can be called from threads spawned by the module. + * + * Note 2: when we unblock a client that is blocked for keys using + * the API RedisModule_BlockClientOnKeys(), the privdata argument here is + * not used, and the reply callback is called with the privdata pointer that + * was passed when blocking the client. + * + * Unblocking a client that was blocked for keys using this API will still + * require the client to get some reply, so the function will use the + * "timeout" handler in order to do so. */ int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) { - pthread_mutex_lock(&moduleUnblockedClientsMutex); - bc->privdata = privdata; - listAddNodeTail(moduleUnblockedClients,bc); - if (bc->client != nullptr) - { - if (write(g_pserver->rgthreadvar[bc->client->iel].module_blocked_pipe[1],"A",1) != 1) { - /* Ignore the error, this is best-effort. */ - } + if (bc->blocked_on_keys) { + /* In theory the user should always pass the timeout handler as an + * argument, but better to be safe than sorry. */ + if (bc->timeout_callback == NULL) return REDISMODULE_ERR; + if (bc->unblocked) return REDISMODULE_OK; + if (bc->client) moduleBlockedClientTimedOut(bc->client); } - else - { - for (int iel = 0; iel < cserver.cthreads; ++iel) - { - if (write(g_pserver->rgthreadvar[iel].module_blocked_pipe[1],"A",1) != 1) { - /* Ignore the error, this is best-effort. */ - } - } - } - - pthread_mutex_unlock(&moduleUnblockedClientsMutex); + moduleUnblockClientByHandle(bc,privdata); return REDISMODULE_OK; } @@ -3777,16 +4614,19 @@ void moduleHandleBlockedClients(int iel) { * touch the shared list. */ /* Call the reply callback if the client is valid and we have - * any callback. */ - if (c && bc->reply_callback) { + * any callback. However the callback is not called if the client + * was blocked on keys (RM_BlockClientOnKeys()), because we already + * called such callback in moduleTryServeClientBlockedOnKey() when + * the key was signaled as ready. */ + if (c && !bc->blocked_on_keys && bc->reply_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; ctx.blocked_privdata = bc->privdata; + ctx.blocked_ready_key = NULL; ctx.module = bc->module; ctx.client = bc->client; ctx.blocked_client = bc; bc->reply_callback(&ctx,(void**)c->argv,c->argc); - moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); } @@ -3834,6 +4674,7 @@ void moduleHandleBlockedClients(int iel) { * referenced in the client blocking context, and must be valid * when calling unblockClient(). */ fastlock_unlock(&c->lock); + bc->module->blocked_clients--; zfree(bc); /* Lock again before to iterate the loop. */ @@ -3878,6 +4719,12 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) { return ctx->blocked_privdata; } +/* Get the key that is ready when the reply callback is called in the context + * of a client blocked by RedisModule_BlockClientOnKeys(). */ +RedisModuleString *RM_GetBlockedClientReadyKey(RedisModuleCtx *ctx) { + return ctx->blocked_ready_key; +} + /* Get the blocked client associated with a given context. * This is useful in the reply and timeout callbacks of blocked clients, * before sometimes the module has the blocked client handle references @@ -3929,10 +4776,10 @@ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) { * access it safely from another thread, so we create a fake client here * in order to keep things like the currently selected database and similar * things. */ - ctx->client = createClient(-1, IDX_EVENT_LOOP_MAIN); + ctx->client = createClient(NULL, IDX_EVENT_LOOP_MAIN); if (bc) { selectDb(ctx->client,bc->dbid); - ctx->client->id = bc->client->id; + if (bc->client) ctx->client->id = bc->client->id; } return ctx; } @@ -4063,6 +4910,20 @@ int RM_SubscribeToKeyspaceEvents(RedisModuleCtx *ctx, int types, RedisModuleNoti return REDISMODULE_OK; } +/* Get the configured bitmap of notify-keyspace-events (Could be used + * for additional filtering in RedisModuleNotificationFunc) */ +int RM_GetNotifyKeyspaceEvents() { + return g_pserver->notify_keyspace_events; +} + +/* Expose notifyKeyspaceEvent to modules */ +int RM_NotifyKeyspaceEvent(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key) { + if (!ctx || !ctx->client) + return REDISMODULE_ERR; + notifyKeyspaceEvent(type, (char *)event, key, ctx->client->db->id); + return REDISMODULE_OK; +} + /* Dispatcher for keyspace notifications to module subscriber functions. * This gets called only if at least one module requested to be notified on * keyspace notifications */ @@ -4309,10 +5170,13 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m UNUSED(ctx); clusterNode *node = clusterLookupNode(id); - if (node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) + if (node == NULL || + node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) + { return REDISMODULE_ERR; + } - if (ip) memcpy(ip,node->name,REDISMODULE_NODE_ID_LEN); + if (ip) strncpy(ip,node->ip,NET_IP_STR_LEN); if (master_id) { /* If the information is not available, the function will set the @@ -4446,7 +5310,7 @@ RedisModuleTimerID RM_CreateTimer(RedisModuleCtx *ctx, mstime_t period, RedisMod timer->module = ctx->module; timer->callback = callback; timer->data = data; - timer->dbid = ctx->client->db->id; + timer->dbid = ctx->client ? ctx->client->db->id : 0; uint64_t expiretime = ustime()+period*1000; uint64_t key; @@ -4518,6 +5382,200 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * Modules ACL API + * + * Implements a hook into the authentication and authorization within Redis. + * --------------------------------------------------------------------------*/ + +/* This function is called when a client's user has changed and invokes the + * client's user changed callback if it was set. This callback should + * cleanup any state the module was tracking about this client. + * + * A client's user can be changed through the AUTH command, module + * authentication, and when a client is freed. */ +void moduleNotifyUserChanged(client *c) { + if (c->auth_callback) { + c->auth_callback(c->id, c->auth_callback_privdata); + + /* The callback will fire exactly once, even if the user remains + * the same. It is expected to completely clean up the state + * so all references are cleared here. */ + c->auth_callback = NULL; + c->auth_callback_privdata = NULL; + c->auth_module = NULL; + } +} + +void revokeClientAuthentication(client *c) { + /* Freeing the client would result in moduleNotifyUserChanged() to be + * called later, however since we use revokeClientAuthentication() also + * in moduleFreeAuthenticatedClients() to implement module unloading, we + * do this action ASAP: this way if the module is unloaded, when the client + * is eventually freed we don't rely on the module to still exist. */ + moduleNotifyUserChanged(c); + + c->puser = DefaultUser; + c->authenticated = 0; + freeClientAsync(c); +} + +/* Cleanup all clients that have been authenticated with this module. This + * is called from onUnload() to give the module a chance to cleanup any + * resources associated with clients it has authenticated. */ +static void moduleFreeAuthenticatedClients(RedisModule *module) { + listIter li; + listNode *ln; + listRewind(g_pserver->clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = (client*)listNodeValue(ln); + if (!c->auth_module) continue; + + RedisModule *auth_module = (RedisModule *) c->auth_module; + if (auth_module == module) { + revokeClientAuthentication(c); + } + } +} + +/* Creates a Redis ACL user that the module can use to authenticate a client. + * After obtaining the user, the module should set what such user can do + * using the RM_SetUserACL() function. Once configured, the user + * can be used in order to authenticate a connection, with the specified + * ACL rules, using the RedisModule_AuthClientWithUser() function. + * + * Note that: + * + * * Users created here are not listed by the ACL command. + * * Users created here are not checked for duplicated name, so it's up to + * the module calling this function to take care of not creating users + * with the same name. + * * The created user can be used to authenticate multiple Redis connections. + * + * The caller can later free the user using the function + * RM_FreeModuleUser(). When this function is called, if there are + * still clients authenticated with this user, they are disconnected. + * The function to free the user should only be used when the caller really + * wants to invalidate the user to define a new one with different + * capabilities. */ +RedisModuleUser *RM_CreateModuleUser(const char *name) { + RedisModuleUser *new_user = (RedisModuleUser*)zmalloc(sizeof(RedisModuleUser)); + new_user->user = ACLCreateUnlinkedUser(); + + /* Free the previous temporarily assigned name to assign the new one */ + sdsfree(new_user->user->name); + new_user->user->name = sdsnew(name); + return new_user; +} + +/* Frees a given user and disconnects all of the clients that have been + * authenticated with it. See RM_CreateModuleUser for detailed usage.*/ +int RM_FreeModuleUser(RedisModuleUser *user) { + ACLFreeUserAndKillClients(user->user); + zfree(user); + return REDISMODULE_OK; +} + +/* Sets the permissions of a user created through the redis module + * interface. The syntax is the same as ACL SETUSER, so refer to the + * documentation in acl.c for more information. See RM_CreateModuleUser + * for detailed usage. + * + * Returns REDISMODULE_OK on success and REDISMODULE_ERR on failure + * and will set an errno describing why the operation failed. */ +int RM_SetModuleUserACL(RedisModuleUser *user, const char* acl) { + return ACLSetUser(user->user, acl, -1); +} + +/* Authenticate the client associated with the context with + * the provided user. Returns REDISMODULE_OK on success and + * REDISMODULE_ERR on error. + * + * This authentication can be tracked with the optional callback and private + * data fields. The callback will be called whenever the user of the client + * changes. This callback should be used to cleanup any state that is being + * kept in the module related to the client authentication. It will only be + * called once, even when the user hasn't changed, in order to allow for a + * new callback to be specified. If this authentication does not need to be + * tracked, pass in NULL for the callback and privdata. + * + * If client_id is not NULL, it will be filled with the id of the client + * that was authenticated. This can be used with the + * RM_DeauthenticateAndCloseClient() API in order to deauthenticate a + * previously authenticated client if the authentication is no longer valid. + * + * For expensive authentication operations, it is recommended to block the + * client and do the authentication in the background and then attach the user + * to the client in a threadsafe context. */ +static int authenticateClientWithUser(RedisModuleCtx *ctx, user *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { + if (user->flags & USER_FLAG_DISABLED) { + return REDISMODULE_ERR; + } + + moduleNotifyUserChanged(ctx->client); + + ctx->client->puser = user; + ctx->client->authenticated = 1; + + if (callback) { + ctx->client->auth_callback = callback; + ctx->client->auth_callback_privdata = privdata; + ctx->client->auth_module = ctx->module; + } + + if (client_id) { + *client_id = ctx->client->id; + } + + return REDISMODULE_OK; +} + + +/* Authenticate the current context's user with the provided redis acl user. + * Returns REDISMODULE_ERR if the user is disabled. + * + * See authenticateClientWithUser for information about callback, client_id, + * and general usage for authentication. */ +int RM_AuthenticateClientWithUser(RedisModuleCtx *ctx, RedisModuleUser *module_user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { + return authenticateClientWithUser(ctx, module_user->user, callback, privdata, client_id); +} + +/* Authenticate the current context's user with the provided redis acl user. + * Returns REDISMODULE_ERR if the user is disabled or the user does not exist. + * + * See authenticateClientWithUser for information about callback, client_id, + * and general usage for authentication. */ +int RM_AuthenticateClientWithACLUser(RedisModuleCtx *ctx, const char *name, size_t len, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { + user *acl_user = ACLGetUserByName(name, len); + + if (!acl_user) { + return REDISMODULE_ERR; + } + return authenticateClientWithUser(ctx, acl_user, callback, privdata, client_id); +} + +/* Deauthenticate and close the client. The client resources will not be + * be immediately freed, but will be cleaned up in a background job. This is + * the recommended way to deauthenicate a client since most clients can't + * handle users becomming deauthenticated. Returns REDISMODULE_ERR when the + * client doesn't exist and REDISMODULE_OK when the operation was successful. + * + * The client ID is returned from the RM_AuthenticateClientWithUser and + * RM_AuthenticateClientWithACLUser APIs, but can be obtained through + * the CLIENT api or through server events. + * + * This function is not thread safe, and must be executed within the context + * of a command or thread safe context. */ +int RM_DeauthenticateAndCloseClient(RedisModuleCtx *ctx, uint64_t client_id) { + UNUSED(ctx); + client *c = lookupClientByID(client_id); + if (c == NULL) return REDISMODULE_ERR; + + /* Revoke also marks client to be closed ASAP */ + revokeClientAuthentication(c); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Modules Dictionary API * @@ -4769,6 +5827,314 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k return res ? REDISMODULE_OK : REDISMODULE_ERR; } + + + +/* -------------------------------------------------------------------------- + * Modules Info fields + * -------------------------------------------------------------------------- */ + +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); + +/* Used to start a new section, before adding any fields. the section name will + * be prefixed by "_" and must only include A-Z,a-z,0-9. + * NULL or empty string indicates the default section (only ) is used. + * When return value is REDISMODULE_ERR, the section should and will be skipped. */ +int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { + sds full_name = sdsdup(ctx->module->name); + if (name != NULL && strlen(name) > 0) + full_name = sdscatfmt(full_name, "_%s", name); + + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + + /* proceed only if: + * 1) no section was requested (emit all) + * 2) the module name was requested (emit all) + * 3) this specific section was requested. */ + if (ctx->requested_section) { + if (strcasecmp(ctx->requested_section, full_name) && + strcasecmp(ctx->requested_section, ctx->module->name)) { + sdsfree(full_name); + ctx->in_section = 0; + return REDISMODULE_ERR; + } + } + if (ctx->sections++) ctx->info = sdscat(ctx->info,"\r\n"); + ctx->info = sdscatfmt(ctx->info, "# %S\r\n", full_name); + ctx->in_section = 1; + sdsfree(full_name); + return REDISMODULE_OK; +} + +/* Starts a dict field, similar to the ones in INFO KEYSPACE. Use normal + * RedisModule_InfoAddField* functions to add the items to this field, and + * terminate with RedisModule_InfoEndDictField. */ +int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) { + if (!ctx->in_section) + return REDISMODULE_ERR; + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + ctx->info = sdscatfmt(ctx->info, + "%s_%s:", + ctx->module->name, + name); + ctx->in_dict_field = 1; + return REDISMODULE_OK; +} + +/* Ends a dict field, see RedisModule_InfoBeginDictField */ +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) { + if (!ctx->in_dict_field) + return REDISMODULE_ERR; + /* trim the last ',' if found. */ + if (ctx->info[sdslen(ctx->info)-1]==',') + sdsIncrLen(ctx->info, -1); + ctx->info = sdscat(ctx->info, "\r\n"); + ctx->in_dict_field = 0; + return REDISMODULE_OK; +} + +/* Used by RedisModuleInfoFunc to add info fields. + * Each field will be automatically prefixed by "_". + * Field names or values must not include \r\n of ":" */ +int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%S,", + field, + (sds)szFromObj(value)); + return REDISMODULE_OK; + } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%S\r\n", + ctx->module->name, + field, + (sds)szFromObj(value)); + return REDISMODULE_OK; +} + +int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%s,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%s\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%.17g,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%.17g\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%I,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%I\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatfmt(ctx->info, + "%s=%U,", + field, + value); + return REDISMODULE_OK; + } + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%U\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { + ctx->module->info_cb = cb; + return REDISMODULE_OK; +} + +sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = (RedisModule*)dictGetVal(de); + if (!module->info_cb) + continue; + RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0}; + module->info_cb(&info_ctx, for_crash_report); + /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ + if (info_ctx.in_dict_field) + RM_InfoEndDictField(&info_ctx); + info = info_ctx.info; + sections = info_ctx.sections; + } + dictReleaseIterator(di); + return info; +} + +/* Get information about the server similar to the one that returns from the + * INFO command. This function takes an optional 'section' argument that may + * be NULL. The return value holds the output and can be used with + * RedisModule_ServerInfoGetField and alike to get the individual fields. + * When done, it needs to be freed with RedisModule_FreeServerInfo or with the + * automatic memory management mechanism if enabled. */ +RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *section) { + struct RedisModuleServerInfoData *d = (RedisModuleServerInfoData*)zmalloc(sizeof(*d)); + d->rax = raxNew(); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_INFO,d); + sds info = genRedisInfoString(section); + int totlines, i; + sds *lines = sdssplitlen(info, sdslen(info), "\r\n", 2, &totlines); + for(i=0; irax,key,keylen,val,NULL)) + sdsfree(val); + } + sdsfree(info); + sdsfreesplitres(lines,totlines); + return d; +} + +/* Free data created with RM_GetServerInfo(). You need to pass the + * context pointer 'ctx' only if the dictionary was created using the + * context instead of passing NULL. */ +void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data) { + if (ctx != NULL) autoMemoryFreed(ctx,REDISMODULE_AM_INFO,data); + raxIterator ri; + raxStart(&ri,data->rax); + while(1) { + raxSeek(&ri,"^",NULL,0); + if (!raxNext(&ri)) break; + raxRemove(data->rax,(unsigned char*)ri.key,ri.key_len,NULL); + sdsfree((sds)ri.data); + } + raxStop(&ri); + raxFree(data->rax); + zfree(data); +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). You + * need to pass the context pointer 'ctx' only if you want to use auto memory + * mechanism to release the returned string. Return value will be NULL if the + * field was not found. */ +RedisModuleString *RM_ServerInfoGetField(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field) { + sds val = (sds)raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) return NULL; + RedisModuleString *o = createStringObject(val,sdslen(val)); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_STRING,o); + return o; +} + +/* Similar to RM_ServerInfoGetField, but returns a char* which should not be freed but the caller. */ +const char *RM_ServerInfoGetFieldC(RedisModuleServerInfoData *data, const char* field) { + sds val = (sds)raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) return NULL; + return val; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not numerical or out of range, return value will be + * 0, and the optional out_err argument will be set to REDISMODULE_ERR. */ +long long RM_ServerInfoGetFieldSigned(RedisModuleServerInfoData *data, const char* field, int *out_err) { + long long ll; + sds val = (sds)raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2ll(val,sdslen(val),&ll)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return ll; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not numerical or out of range, return value will be + * 0, and the optional out_err argument will be set to REDISMODULE_ERR. */ +unsigned long long RM_ServerInfoGetFieldUnsigned(RedisModuleServerInfoData *data, const char* field, int *out_err) { + unsigned long long ll; + sds val = (sds)raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2ull(val,&ll)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return ll; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not a double, return value will be 0, and the + * optional out_err argument will be set to REDISMODULE_ERR. */ +double RM_ServerInfoGetFieldDouble(RedisModuleServerInfoData *data, const char* field, int *out_err) { + double dbl; + sds val = (sds)raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2d(val,sdslen(val),&dbl)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return dbl; +} + /* -------------------------------------------------------------------------- * Modules utility APIs * -------------------------------------------------------------------------- */ @@ -4805,7 +6171,7 @@ void RM_GetRandomHexChars(char *dst, size_t len) { * lifetime. The API relies on the fact that it will always be valid in * the future. */ int RM_ExportSharedAPI(RedisModuleCtx *ctx, const char *apiname, void *func) { - RedisModuleSharedAPI *sapi = (RedisModuleSharedAPI*)zmalloc(sizeof(*sapi), MALLOC_LOCAL); + RedisModuleSharedAPI *sapi = (RedisModuleSharedAPI*)zmalloc(sizeof(*sapi)); sapi->module = ctx->module; sapi->func = func; if (dictAdd(g_pserver->sharedapi, (char*)apiname, sapi) != DICT_OK) { @@ -5003,11 +6369,13 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi ln = listSearchKey(moduleCommandFilters,filter); if (!ln) return REDISMODULE_ERR; listDelNode(moduleCommandFilters,ln); - + ln = listSearchKey(ctx->module->filters,filter); if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); + zfree(filter); + return REDISMODULE_OK; } @@ -5114,6 +6482,716 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } +/* For a given pointer allocated via RedisModule_Alloc() or + * RedisModule_Realloc(), return the amount of memory allocated for it. + * Note that this may be different (larger) than the memory we allocated + * with the allocation calls, since sometimes the underlying allocator + * will allocate more memory. + */ +size_t RM_MallocSize(void* ptr){ + return zmalloc_size(ptr); +} + +/* Return the a number between 0 to 1 indicating the amount of memory + * currently used, relative to the Redis "maxmemory" configuration. + * + * 0 - No memory limit configured. + * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range. + * Exactly 1 - Memory limit reached. + * Greater 1 - More memory used than the configured limit. + */ +float RM_GetUsedMemoryRatio(){ + float level; + getMaxmemoryState(NULL, NULL, NULL, &level); + return level; +} + +/* -------------------------------------------------------------------------- + * Scanning keyspace and hashes + * -------------------------------------------------------------------------- */ + +typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); +typedef struct { + RedisModuleCtx *ctx; + void* user_data; + RedisModuleScanCB fn; +} ScanCBData; + +typedef struct RedisModuleScanCursor{ + int cursor; + int done; +}RedisModuleScanCursor; + +static void moduleScanCallback(void *privdata, const dictEntry *de) { + ScanCBData *data = (ScanCBData*)privdata; + sds key = (sds)dictGetKey(de); + robj* val = (robj*)dictGetVal(de); + RedisModuleString *keyname = createObject(OBJ_STRING,sdsdup(key)); + + /* Setup the key handle. */ + RedisModuleKey kp = {0}; + moduleInitKey(&kp, data->ctx, keyname, val, REDISMODULE_READ); + + data->fn(data->ctx, keyname, &kp, data->user_data); + + moduleCloseKey(&kp); + decrRefCount(keyname); +} + +/* Create a new cursor to be used with RedisModule_Scan */ +RedisModuleScanCursor *RM_ScanCursorCreate() { + RedisModuleScanCursor* cursor = (RedisModuleScanCursor*)zmalloc(sizeof(*cursor)); + cursor->cursor = 0; + cursor->done = 0; + return cursor; +} + +/* Restart an existing cursor. The keys will be rescanned. */ +void RM_ScanCursorRestart(RedisModuleScanCursor *cursor) { + cursor->cursor = 0; + cursor->done = 0; +} + +/* Destroy the cursor struct. */ +void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { + zfree(cursor); +} + +/* Scan api that allows a module to scan all the keys and value in the selected db. + * + * Callback for scan implementation. + * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); + * - ctx - the redis module context provided to for the scan. + * - keyname - owned by the caller and need to be retained if used after this function. + * - key - holds info on the key and value, it is provided as best effort, in some cases it might + * be NULL, in which case the user should (can) use RedisModule_OpenKey (and CloseKey too). + * when it is provided, it is owned by the caller and will be free when the callback returns. + * - privdata - the user data provided to RedisModule_Scan. + * + * The way it should be used: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * while(RedisModule_Scan(ctx, c, callback, privateData)); + * RedisModule_ScanCursorDestroy(c); + * + * It is also possible to use this API from another thread while the lock is acquired durring + * the actuall call to RM_Scan: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModule_ThreadSafeContextLock(ctx); + * while(RedisModule_Scan(ctx, c, callback, privateData)){ + * RedisModule_ThreadSafeContextUnlock(ctx); + * // do some background job + * RedisModule_ThreadSafeContextLock(ctx); + * } + * RedisModule_ScanCursorDestroy(c); + * + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. */ +int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) { + if (cursor->done) { + errno = ENOENT; + return 0; + } + int ret = 1; + ScanCBData data = { ctx, privdata, fn }; + cursor->cursor = dictScan(ctx->client->db->pdict, cursor->cursor, moduleScanCallback, NULL, &data); + if (cursor->cursor == 0) { + cursor->done = 1; + ret = 0; + } + errno = 0; + return ret; +} + +typedef void (*RedisModuleScanKeyCB)(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata); +typedef struct { + RedisModuleKey *key; + void* user_data; + RedisModuleScanKeyCB fn; +} ScanKeyCBData; + +static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { + ScanKeyCBData *data = (ScanKeyCBData*)privdata; + sds key = (sds)dictGetKey(de); + robj *o = data->key->value; + robj *field = createStringObject(key, sdslen(key)); + robj *value = NULL; + if (o->type == OBJ_SET) { + value = NULL; + } else if (o->type == OBJ_HASH) { + sds val = (sds)dictGetVal(de); + value = createStringObject(val, sdslen(val)); + } else if (o->type == OBJ_ZSET) { + double *val = (double*)dictGetVal(de); + value = createStringObjectFromLongDouble(*val, 0); + } + + data->fn(data->key, field, value, data->user_data); + decrRefCount(field); + if (value) decrRefCount(value); +} + +/* Scan api that allows a module to scan the elements in a hash, set or sorted set key + * + * Callback for scan implementation. + * void scan_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata); + * - key - the redis key context provided to for the scan. + * - field - field name, owned by the caller and need to be retained if used + * after this function. + * - value - value string or NULL for set type, owned by the caller and need to + * be retained if used after this function. + * - privdata - the user data provided to RedisModule_ScanKey. + * + * The way it should be used: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * while(RedisModule_ScanKey(key, c, callback, privateData)); + * RedisModule_CloseKey(key); + * RedisModule_ScanCursorDestroy(c); + * + * It is also possible to use this API from another thread while the lock is acquired durring + * the actuall call to RM_Scan, and re-opening the key each time: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModule_ThreadSafeContextLock(ctx); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * while(RedisModule_ScanKey(ctx, c, callback, privateData)){ + * RedisModule_CloseKey(key); + * RedisModule_ThreadSafeContextUnlock(ctx); + * // do some background job + * RedisModule_ThreadSafeContextLock(ctx); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * } + * RedisModule_CloseKey(key); + * RedisModule_ScanCursorDestroy(c); + * + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. */ +int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) { + if (key == NULL || key->value == NULL) { + errno = EINVAL; + return 0; + } + dict *ht = NULL; + robj *o = key->value; + if (o->type == OBJ_SET) { + if (o->encoding == OBJ_ENCODING_HT) + ht = (dict*)ptrFromObj(o); + } else if (o->type == OBJ_HASH) { + if (o->encoding == OBJ_ENCODING_HT) + ht = (dict*)ptrFromObj(o); + } else if (o->type == OBJ_ZSET) { + if (o->encoding == OBJ_ENCODING_SKIPLIST) + ht = ((zset *)ptrFromObj(o))->pdict; + } else { + errno = EINVAL; + return 0; + } + if (cursor->done) { + errno = ENOENT; + return 0; + } + int ret = 1; + if (ht) { + ScanKeyCBData data = { key, privdata, fn }; + cursor->cursor = dictScan(ht, cursor->cursor, moduleScanKeyCallback, NULL, &data); + if (cursor->cursor == 0) { + cursor->done = 1; + ret = 0; + } + } else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_INTSET) { + int pos = 0; + int64_t ll; + while(intsetGet((intset*)ptrFromObj(o),pos++,&ll)) { + robj *field = createStringObjectFromLongLong(ll); + fn(key, field, NULL, privdata); + decrRefCount(field); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { + unsigned char *p = ziplistIndex((unsigned char*)ptrFromObj(o),0); + unsigned char *vstr; + unsigned int vlen; + long long vll; + while(p) { + ziplistGet(p,&vstr,&vlen,&vll); + robj *field = (vstr != NULL) ? + createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll); + p = ziplistNext((unsigned char*)ptrFromObj(o),p); + ziplistGet(p,&vstr,&vlen,&vll); + robj *value = (vstr != NULL) ? + createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll); + fn(key, field, value, privdata); + p = ziplistNext((unsigned char*)ptrFromObj(o),p); + decrRefCount(field); + decrRefCount(value); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } + errno = 0; + return ret; +} + + +/* -------------------------------------------------------------------------- + * Module fork API + * -------------------------------------------------------------------------- */ + +/* Create a background child process with the current frozen snaphost of the + * main process where you can do some processing in the background without + * affecting / freezing the traffic and no need for threads and GIL locking. + * Note that Redis allows for only one concurrent fork. + * When the child wants to exit, it should call RedisModule_ExitFromChild. + * If the parent wants to kill the child it should call RedisModule_KillForkChild + * The done handler callback will be executed on the parent process when the + * child existed (but not when killed) + * Return: -1 on failure, on success the parent process will get a positive PID + * of the child, and the child process will get 0. + */ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { + pid_t childpid; + if (hasActiveChildProcess()) { + return -1; + } + + openChildInfoPipe(); + if ((childpid = redisFork()) == 0) { + /* Child */ + redisSetProcTitle("redis-module-fork"); + } else if (childpid == -1) { + closeChildInfoPipe(); + serverLog(LL_WARNING,"Can't fork for module: %s", strerror(errno)); + } else { + /* Parent */ + g_pserver->module_child_pid = childpid; + moduleForkInfo.done_handler = cb; + moduleForkInfo.done_handler_user_data = user_data; + serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid); + } + return childpid; +} + +/* Call from the child process when you want to terminate it. + * retcode will be provided to the done handler executed on the parent process. + */ +int RM_ExitFromChild(int retcode) { + sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); + exitFromChild(retcode); + return REDISMODULE_OK; +} + +/* Kill the active module forked child, if there is one active and the + * pid matches, and returns C_OK. Otherwise if there is no active module + * child or the pid does not match, return C_ERR without doing anything. */ +int TerminateModuleForkChild(int child_pid, int wait) { + /* Module child should be active and pid should match. */ + if (g_pserver->module_child_pid == -1 || + g_pserver->module_child_pid != child_pid) return C_ERR; + + int statloc; + serverLog(LL_NOTICE,"Killing running module fork child: %ld", + (long) g_pserver->module_child_pid); + if (kill(g_pserver->module_child_pid,SIGUSR1) != -1 && wait) { + while(wait4(g_pserver->module_child_pid,&statloc,0,NULL) != + g_pserver->module_child_pid); + } + /* Reset the buffer accumulating changes while the child saves. */ + g_pserver->module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; + closeChildInfoPipe(); + updateDictResizePolicy(); + return C_OK; +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) { + /* Kill module child, wait for child exit. */ + if (TerminateModuleForkChild(child_pid,1) == C_OK) + return REDISMODULE_OK; + else + return REDISMODULE_ERR; +} + +void ModuleForkDoneHandler(int exitcode, int bysignal) { + serverLog(LL_NOTICE, + "Module fork exited pid: %d, retcode: %d, bysignal: %d", + g_pserver->module_child_pid, exitcode, bysignal); + if (moduleForkInfo.done_handler) { + moduleForkInfo.done_handler(exitcode, bysignal, + moduleForkInfo.done_handler_user_data); + } + g_pserver->module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; +} + +/* -------------------------------------------------------------------------- + * Server hooks implementation + * -------------------------------------------------------------------------- */ + +/* Register to be notified, via a callback, when the specified server event + * happens. The callback is called with the event as argument, and an additional + * argument which is a void pointer and should be cased to a specific type + * that is event-specific (but many events will just use NULL since they do not + * have additional information to pass to the callback). + * + * If the callback is NULL and there was a previous subscription, the module + * will be unsubscribed. If there was a previous subscription and the callback + * is not null, the old callback will be replaced with the new one. + * + * The callback must be of this type: + * + * int (*RedisModuleEventCallback)(RedisModuleCtx *ctx, + * RedisModuleEvent eid, + * uint64_t subevent, + * void *data); + * + * The 'ctx' is a normal Redis module context that the callback can use in + * order to call other modules APIs. The 'eid' is the event itself, this + * is only useful in the case the module subscribed to multiple events: using + * the 'id' field of this structure it is possible to check if the event + * is one of the events we registered with this callback. The 'subevent' field + * depends on the event that fired. + * + * Finally the 'data' pointer may be populated, only for certain events, with + * more relevant data. + * + * Here is a list of events you can use as 'eid' and related sub events: + * + * RedisModuleEvent_ReplicationRoleChanged + * + * This event is called when the instance switches from master + * to replica or the other way around, however the event is + * also called when the replica remains a replica but starts to + * replicate with a different master. + * + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER + * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA + * + * The 'data' field can be casted by the callback to a + * RedisModuleReplicationInfo structure with the following fields: + * + * int master; // true if master, false if replica + * char *masterhost; // master instance hostname for NOW_REPLICA + * int masterport; // master instance port for NOW_REPLICA + * char *replid1; // Main replication ID + * char *replid2; // Secondary replication ID + * uint64_t repl1_offset; // Main replication offset + * uint64_t repl2_offset; // Offset of replid2 validity + * + * RedisModuleEvent_Persistence + * + * This event is called when RDB saving or AOF rewriting starts + * and ends. The following sub events are available: + * + * REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START + * REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START + * REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START + * REDISMODULE_SUBEVENT_PERSISTENCE_ENDED + * REDISMODULE_SUBEVENT_PERSISTENCE_FAILED + * + * The above events are triggered not just when the user calls the + * relevant commands like BGSAVE, but also when a saving operation + * or AOF rewriting occurs because of internal server triggers. + * The SYNC_RDB_START sub events are happening in the forground due to + * SAVE command, FLUSHALL, or server shutdown, and the other RDB and + * AOF sub events are executed in a background fork child, so any + * action the module takes can only affect the generated AOF or RDB, + * but will not be reflected in the parent process and affect connected + * clients and commands. Also note that the AOF_START sub event may end + * up saving RDB content in case of an AOF with rdb-preamble. + * + * RedisModuleEvent_FlushDB + * + * The FLUSHALL, FLUSHDB or an internal flush (for instance + * because of replication, after the replica synchronization) + * happened. The following sub events are available: + * + * REDISMODULE_SUBEVENT_FLUSHDB_START + * REDISMODULE_SUBEVENT_FLUSHDB_END + * + * The data pointer can be casted to a RedisModuleFlushInfo + * structure with the following fields: + * + * int32_t async; // True if the flush is done in a thread. + * See for instance FLUSHALL ASYNC. + * In this case the END callback is invoked + * immediately after the database is put + * in the free list of the thread. + * int32_t dbnum; // Flushed database number, -1 for all the DBs + * in the case of the FLUSHALL operation. + * + * The start event is called *before* the operation is initated, thus + * allowing the callback to call DBSIZE or other operation on the + * yet-to-free keyspace. + * + * RedisModuleEvent_Loading + * + * Called on loading operations: at startup when the server is + * started, but also after a first synchronization when the + * replica is loading the RDB file from the master. + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_LOADING_RDB_START + * REDISMODULE_SUBEVENT_LOADING_AOF_START + * REDISMODULE_SUBEVENT_LOADING_REPL_START + * REDISMODULE_SUBEVENT_LOADING_ENDED + * REDISMODULE_SUBEVENT_LOADING_FAILED + * + * Note that AOF loading may start with an RDB data in case of + * rdb-preamble, in which case you'll only recieve an AOF_START event. + * + * + * RedisModuleEvent_ClientChange + * + * Called when a client connects or disconnects. + * The data pointer can be casted to a RedisModuleClientInfo + * structure, documented in RedisModule_GetClientInfoById(). + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED + * REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED + * + * RedisModuleEvent_Shutdown + * + * The server is shutting down. No subevents are available. + * + * RedisModuleEvent_ReplicaChange + * + * This event is called when the instance (that can be both a + * master or a replica) get a new online replica, or lose a + * replica since it gets disconnected. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE + * REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE + * + * No additional information is available so far: future versions + * of Redis will have an API in order to enumerate the replicas + * connected and their state. + * + * RedisModuleEvent_CronLoop + * + * This event is called every time Redis calls the serverCron() + * function in order to do certain bookkeeping. Modules that are + * required to do operations from time to time may use this callback. + * Normally Redis calls this function 10 times per second, but + * this changes depending on the "hz" configuration. + * No sub events are available. + * + * The data pointer can be casted to a RedisModuleCronLoop + * structure with the following fields: + * + * int32_t hz; // Approximate number of events per second. + * + * RedisModuleEvent_MasterLinkChange + * + * This is called for replicas in order to notify when the + * replication link becomes functional (up) with our master, + * or when it goes down. Note that the link is not considered + * up when we just connected to the master, but only if the + * replication is happening correctly. + * The following sub events are available: + * + * REDISMODULE_SUBEVENT_MASTER_LINK_UP + * REDISMODULE_SUBEVENT_MASTER_LINK_DOWN + * + * RedisModuleEvent_ModuleChange + * + * This event is called when a new module is loaded or one is unloaded. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_MODULE_LOADED + * REDISMODULE_SUBEVENT_MODULE_UNLOADED + * + * The data pointer can be casted to a RedisModuleModuleChange + * structure with the following fields: + * + * const char* module_name; // Name of module loaded or unloaded. + * int32_t module_version; // Module version. + * + * RedisModuleEvent_LoadingProgress + * + * This event is called repeatedly called while an RDB or AOF file + * is being loaded. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB + * REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF + * + * The data pointer can be casted to a RedisModuleLoadingProgress + * structure with the following fields: + * + * int32_t hz; // Approximate number of events per second. + * int32_t progress; // Approximate progress between 0 and 1024, + * or -1 if unknown. + * + * The function returns REDISMODULE_OK if the module was successfully subscrived + * for the specified event. If the API is called from a wrong context then + * REDISMODULE_ERR is returned. */ +int RM_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) { + RedisModuleEventListener *el; + + /* Protect in case of calls from contexts without a module reference. */ + if (ctx->module == NULL) return REDISMODULE_ERR; + + /* Search an event matching this module and event ID. */ + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + while((ln = listNext(&li))) { + el = (RedisModuleEventListener*)ln->value; + if (el->module == ctx->module && el->event.id == event.id) + break; /* Matching event found. */ + } + + /* Modify or remove the event listener if we already had one. */ + if (ln) { + if (callback == NULL) { + listDelNode(RedisModule_EventListeners,ln); + zfree(el); + } else { + el->callback = callback; /* Update the callback with the new one. */ + } + return REDISMODULE_OK; + } + + /* No event found, we need to add a new one. */ + el = (RedisModuleEventListener*)zmalloc(sizeof(*el)); + el->module = ctx->module; + el->event = event; + el->callback = callback; + listAddNodeTail(RedisModule_EventListeners,el); + return REDISMODULE_OK; +} + +/* This is called by the Redis internals every time we want to fire an + * event that can be interceppted by some module. The pointer 'data' is useful + * in order to populate the event-specific structure when needed, in order + * to return the structure with more information to the callback. + * + * 'eid' and 'subid' are just the main event ID and the sub event associated + * with the event, depending on what exactly happened. */ +void moduleFireServerEvent(uint64_t eid, int subid, void *data) { + /* Fast path to return ASAP if there is nothing to do, avoiding to + * setup the iterator and so forth: we want this call to be extremely + * cheap if there are no registered modules. */ + if (listLength(RedisModule_EventListeners) == 0) return; + + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + while((ln = listNext(&li))) { + RedisModuleEventListener *el = (RedisModuleEventListener*)ln->value; + if (el->event.id == eid) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = el->module; + + if (ModulesInHooks == 0) { + ctx.client = moduleFreeContextReusedClient; + } else { + ctx.client = createClient(NULL, IDX_EVENT_LOOP_MAIN); + ctx.client->flags |= CLIENT_MODULE; + ctx.client->puser = NULL; /* Root user. */ + } + + void *moduledata = NULL; + RedisModuleClientInfoV1 civ1; + RedisModuleReplicationInfoV1 riv1; + RedisModuleModuleChangeV1 mcv1; + /* Start at DB zero by default when calling the handler. It's + * up to the specific event setup to change it when it makes + * sense. For instance for FLUSHDB events we select the correct + * DB automatically. */ + selectDb(ctx.client, 0); + + /* Event specific context and data pointer setup. */ + if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) { + modulePopulateClientInfoStructure(&civ1,(client*)data, + el->event.dataver); + moduledata = &civ1; + } else if (eid == REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED) { + modulePopulateReplicationInfoStructure(&riv1,el->event.dataver); + moduledata = &riv1; + } else if (eid == REDISMODULE_EVENT_FLUSHDB) { + moduledata = data; + RedisModuleFlushInfoV1 *fi = (RedisModuleFlushInfoV1*)data; + if (fi->dbnum != -1) + selectDb(ctx.client, fi->dbnum); + } else if (eid == REDISMODULE_EVENT_MODULE_CHANGE) { + RedisModule *m = (RedisModule*)data; + if (m == el->module) + continue; + mcv1.version = REDISMODULE_MODULE_CHANGE_VERSION; + mcv1.module_name = m->name; + mcv1.module_version = m->ver; + moduledata = &mcv1; + } else if (eid == REDISMODULE_EVENT_LOADING_PROGRESS) { + moduledata = data; + } else if (eid == REDISMODULE_EVENT_CRON_LOOP) { + moduledata = data; + } + + ModulesInHooks++; + el->module->in_hook++; + el->callback(&ctx,el->event,subid,moduledata); + el->module->in_hook--; + ModulesInHooks--; + + if (ModulesInHooks != 0) freeClient(ctx.client); + moduleFreeContext(&ctx); + } + } +} + +/* Remove all the listeners for this module: this is used before unloading + * a module. */ +void moduleUnsubscribeAllServerEvents(RedisModule *module) { + RedisModuleEventListener *el; + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + + while((ln = listNext(&li))) { + el = (RedisModuleEventListener*)ln->value; + if (el->module == module) { + listDelNode(RedisModule_EventListeners,ln); + zfree(el); + } + } +} + +void processModuleLoadingProgressEvent(int is_aof) { + long long now = ustime(); + static long long next_event = 0; + if (now >= next_event) { + /* Fire the loading progress modules end event. */ + int progress = -1; + if (g_pserver->loading_total_bytes) + progress = (g_pserver->loading_total_bytes<<10) / g_pserver->loading_total_bytes; + RedisModuleFlushInfoV1 fi = {REDISMODULE_LOADING_PROGRESS_VERSION, + g_pserver->hz, + progress}; + moduleFireServerEvent(REDISMODULE_EVENT_LOADING_PROGRESS, + is_aof? + REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF: + REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB, + &fi); + /* decide when the next event should fire. */ + next_event = now + 1000000 / g_pserver->hz; + } +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -5156,7 +7234,7 @@ void moduleInitModulesSystem(void) { /* Set up the keyspace notification susbscriber list and static client */ moduleKeyspaceSubscribers = listCreate(); - moduleFreeContextReusedClient = createClient(-1, IDX_EVENT_LOOP_MAIN); + moduleFreeContextReusedClient = createClient(NULL, IDX_EVENT_LOOP_MAIN); moduleFreeContextReusedClient->flags |= CLIENT_MODULE; moduleFreeContextReusedClient->puser = NULL; /* root user. */ @@ -5168,6 +7246,9 @@ void moduleInitModulesSystem(void) { /* Create the timers radix tree. */ Timers = raxNew(); + /* Setup the event listeners data structures. */ + RedisModule_EventListeners = listCreate(); + /* Our thread-safe contexts GIL must start with already locked: * it is just unlocked when it's safe. */ moduleAcquireGIL(true); @@ -5203,6 +7284,8 @@ void moduleLoadFromQueue(void) { void moduleFreeModuleStructure(struct RedisModule *module) { listRelease(module->types); listRelease(module->filters); + listRelease(module->usedby); + listRelease(module->usingMods); sdsfree(module->name); zfree(module); } @@ -5235,7 +7318,9 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { int (*onload)(void *, void **, int); void *handle; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; - + ctx.client = moduleFreeContextReusedClient; + selectDb(ctx.client, 0); + struct stat st; if (stat(path, &st) == 0) { // this check is best effort @@ -5273,8 +7358,14 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { /* Redis module loaded! Register it. */ dictAdd(modules,ctx.module->name,ctx.module); + ctx.module->blocked_clients = 0; ctx.module->handle = handle; serverLog(LL_NOTICE,"Module '%s' loaded from %s",ctx.module->name,path); + /* Fire the loaded modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE, + REDISMODULE_SUBEVENT_MODULE_LOADED, + ctx.module); + moduleFreeContext(&ctx); return C_OK; } @@ -5298,8 +7389,29 @@ int moduleUnload(sds name) { } else if (listLength(module->usedby)) { errno = EPERM; return REDISMODULE_ERR; + } else if (module->blocked_clients) { + errno = EAGAIN; + return REDISMODULE_ERR; } + /* Give module a chance to clean up. */ + int (*onunload)(void *); + onunload = (int (*)(void *))(unsigned long) dlsym(module->handle, "RedisModule_OnUnload"); + if (onunload) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = module; + ctx.client = moduleFreeContextReusedClient; + int unload_status = onunload((void*)&ctx); + moduleFreeContext(&ctx); + + if (unload_status == REDISMODULE_ERR) { + serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", name); + errno = ECANCELED; + return REDISMODULE_ERR; + } + } + + moduleFreeAuthenticatedClients(module); moduleUnregisterCommands(module); moduleUnregisterSharedAPI(module); moduleUnregisterUsedAPI(module); @@ -5307,8 +7419,7 @@ int moduleUnload(sds name) { /* Remove any notification subscribers this module might have */ moduleUnsubscribeNotifications(module); - - /* Unregister all the hooks. TODO: Yet no hooks support here. */ + moduleUnsubscribeAllServerEvents(module); /* Unload the dynamic library. */ if (dlclose(module->handle) == -1) { @@ -5318,6 +7429,11 @@ int moduleUnload(sds name) { module->name, error); } + /* Fire the unloaded modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE, + REDISMODULE_SUBEVENT_MODULE_UNLOADED, + module); + /* Remove from list of modules. */ serverLog(LL_NOTICE,"Module %s unloaded",module->name); dictDelete(modules,module->name); @@ -5346,6 +7462,62 @@ void addReplyLoadedModules(client *c) { dictReleaseIterator(di); } +/* Helper for genModulesInfoString(): given a list of modules, return + * am SDS string in the form "[modulename|modulename2|...]" */ +sds genModulesInfoStringRenderModulesList(list *l) { + listIter li; + listNode *ln; + listRewind(l,&li); + sds output = sdsnew("["); + while((ln = listNext(&li))) { + RedisModule *module = (RedisModule*)ln->value; + output = sdscat(output,module->name); + } + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + +/* Helper for genModulesInfoString(): render module options as an SDS string. */ +sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) { + sds output = sdsnew("["); + if (module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) + output = sdscat(output,"handle-io-errors|"); + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + + +/* Helper function for the INFO command: adds loaded modules as to info's + * output. + * + * After the call, the passed sds info string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds genModulesInfoString(sds info) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + sds name = (sds)dictGetKey(de); + struct RedisModule *module = (RedisModule*)dictGetVal(de); + + sds usedby = genModulesInfoStringRenderModulesList(module->usedby); + sds usingMods = genModulesInfoStringRenderModulesList(module->usingMods); + sds options = genModulesInfoStringRenderModuleOptions(module); + info = sdscatfmt(info, + "module:name=%S,ver=%i,api=%i,filters=%i," + "usedby=%S,using=%S,options=%S\r\n", + name, module->ver, module->apiver, + (int)listLength(module->filters), usedby, usingMods, options); + sdsfree(usedby); + sdsfree(usingMods); + sdsfree(options); + } + dictReleaseIterator(di); + return info; +} + /* Redis MODULE command. * * MODULE LOAD [args...] */ @@ -5391,6 +7563,10 @@ NULL errmsg = "the module exports APIs used by other modules. " "Please unload them first and try again"; break; + case EAGAIN: + errmsg = "the module has blocked clients. " + "Please wait them unblocked and try again"; + break; default: errmsg = "operation not possible."; break; @@ -5410,6 +7586,90 @@ size_t moduleCount(void) { return dictSize(modules); } +/* Set the key last access time for LRU based eviction. not relevent if the + * servers's maxmemory policy is LFU based. Value is idle time in milliseconds. + * returns REDISMODULE_OK if the LRU was updated, REDISMODULE_ERR otherwise. */ +int RM_SetLRU(RedisModuleKey *key, mstime_t lru_idle) { + if (!key->value) + return REDISMODULE_ERR; + if (objectSetLRUOrLFU(key->value, -1, lru_idle, lru_idle>=0 ? LRU_CLOCK() : 0, 1)) + return REDISMODULE_OK; + return REDISMODULE_ERR; +} + +/* Gets the key last access time. + * Value is idletime in milliseconds or -1 if the server's eviction policy is + * LFU based. + * returns REDISMODULE_OK if when key is valid. */ +int RM_GetLRU(RedisModuleKey *key, mstime_t *lru_idle) { + *lru_idle = -1; + if (!key->value) + return REDISMODULE_ERR; + if (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU) + return REDISMODULE_OK; + *lru_idle = estimateObjectIdleTime(key->value); + return REDISMODULE_OK; +} + +/* Set the key access frequency. only relevant if the server's maxmemory policy + * is LFU based. + * The frequency is a logarithmic counter that provides an indication of + * the access frequencyonly (must be <= 255). + * returns REDISMODULE_OK if the LFU was updated, REDISMODULE_ERR otherwise. */ +int RM_SetLFU(RedisModuleKey *key, long long lfu_freq) { + if (!key->value) + return REDISMODULE_ERR; + if (objectSetLRUOrLFU(key->value, lfu_freq, -1, 0, 1)) + return REDISMODULE_OK; + return REDISMODULE_ERR; +} + +/* Gets the key access frequency or -1 if the server's eviction policy is not + * LFU based. + * returns REDISMODULE_OK if when key is valid. */ +int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) { + *lfu_freq = -1; + if (!key->value) + return REDISMODULE_ERR; + if (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU) + *lfu_freq = LFUDecrAndReturn(key->value); + return REDISMODULE_OK; +} + +/* Replace the value assigned to a module type. + * + * The key must be open for writing, have an existing value, and have a moduleType + * that matches the one specified by the caller. + * + * Unlike RM_ModuleTypeSetValue() which will free the old value, this function + * simply swaps the old value with the new value. + * + * The function returns REDISMODULE_OK on success, REDISMODULE_ERR on errors + * such as: + * + * 1. Key is not opened for writing. + * 2. Key is not a module data type key. + * 3. Key is a module datatype other than 'mt'. + * + * If old_value is non-NULL, the old value is returned by reference. + */ +int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_value, void **old_value) { + if (!(key->mode & REDISMODULE_WRITE) || key->iter) + return REDISMODULE_ERR; + if (!key->value || key->value->type != OBJ_MODULE) + return REDISMODULE_ERR; + + moduleValue *mv = (moduleValue*)ptrFromObj(key->value); + if (mv->type != mt) + return REDISMODULE_ERR; + + if (old_value) + *old_value = mv->value; + mv->value = new_value; + + return REDISMODULE_OK; +} + /* Register all the APIs we export. Keep this function at the end of the * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { @@ -5428,12 +7688,18 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ReplyWithError); REGISTER_API(ReplyWithSimpleString); REGISTER_API(ReplyWithArray); + REGISTER_API(ReplyWithNullArray); + REGISTER_API(ReplyWithEmptyArray); REGISTER_API(ReplySetArrayLength); REGISTER_API(ReplyWithString); + REGISTER_API(ReplyWithEmptyString); + REGISTER_API(ReplyWithVerbatimString); REGISTER_API(ReplyWithStringBuffer); + REGISTER_API(ReplyWithCString); REGISTER_API(ReplyWithNull); REGISTER_API(ReplyWithCallReply); REGISTER_API(ReplyWithDouble); + REGISTER_API(ReplyWithLongDouble); REGISTER_API(GetSelectedDb); REGISTER_API(SelectDb); REGISTER_API(OpenKey); @@ -5444,6 +7710,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ListPop); REGISTER_API(StringToLongLong); REGISTER_API(StringToDouble); + REGISTER_API(StringToLongDouble); REGISTER_API(Call); REGISTER_API(CallReplyProto); REGISTER_API(FreeCallReply); @@ -5455,6 +7722,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateStringFromCallReply); REGISTER_API(CreateString); REGISTER_API(CreateStringFromLongLong); + REGISTER_API(CreateStringFromLongDouble); REGISTER_API(CreateStringFromString); REGISTER_API(CreateStringPrintf); REGISTER_API(FreeString); @@ -5469,6 +7737,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(StringTruncate); REGISTER_API(SetExpire); REGISTER_API(GetExpire); + REGISTER_API(ResetDataset); + REGISTER_API(DbSize); + REGISTER_API(RandomKey); REGISTER_API(ZsetAdd); REGISTER_API(ZsetIncrby); REGISTER_API(ZsetScore); @@ -5488,11 +7759,16 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(KeyAtPos); REGISTER_API(GetClientId); REGISTER_API(GetContextFlags); + REGISTER_API(AvoidReplicaTraffic); REGISTER_API(PoolAlloc); REGISTER_API(CreateDataType); REGISTER_API(ModuleTypeSetValue); + REGISTER_API(ModuleTypeReplaceValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); + REGISTER_API(IsIOError); + REGISTER_API(SetModuleOptions); + REGISTER_API(SignalModifiedKey); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); @@ -5505,14 +7781,21 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(LoadDouble); REGISTER_API(SaveFloat); REGISTER_API(LoadFloat); + REGISTER_API(SaveLongDouble); + REGISTER_API(LoadLongDouble); + REGISTER_API(SaveDataTypeToString); + REGISTER_API(LoadDataTypeFromString); REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); + REGISTER_API(_Assert); + REGISTER_API(LatencyAddSample); REGISTER_API(StringAppendBuffer); REGISTER_API(RetainString); REGISTER_API(StringCompare); REGISTER_API(GetContextFromIO); REGISTER_API(GetKeyNameFromIO); + REGISTER_API(GetKeyNameFromModuleKey); REGISTER_API(BlockClient); REGISTER_API(UnblockClient); REGISTER_API(IsBlockedReplyRequest); @@ -5527,6 +7810,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DigestAddStringBuffer); REGISTER_API(DigestAddLongLong); REGISTER_API(DigestEndSequence); + REGISTER_API(NotifyKeyspaceEvent); + REGISTER_API(GetNotifyKeyspaceEvents); REGISTER_API(SubscribeToKeyspaceEvents); REGISTER_API(RegisterClusterMessageReceiver); REGISTER_API(SendClusterMessage); @@ -5575,4 +7860,46 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); + REGISTER_API(Fork); + REGISTER_API(ExitFromChild); + REGISTER_API(KillForkChild); + REGISTER_API(RegisterInfoFunc); + REGISTER_API(InfoAddSection); + REGISTER_API(InfoBeginDictField); + REGISTER_API(InfoEndDictField); + REGISTER_API(InfoAddFieldString); + REGISTER_API(InfoAddFieldCString); + REGISTER_API(InfoAddFieldDouble); + REGISTER_API(InfoAddFieldLongLong); + REGISTER_API(InfoAddFieldULongLong); + REGISTER_API(GetServerInfo); + REGISTER_API(FreeServerInfo); + REGISTER_API(ServerInfoGetField); + REGISTER_API(ServerInfoGetFieldC); + REGISTER_API(ServerInfoGetFieldSigned); + REGISTER_API(ServerInfoGetFieldUnsigned); + REGISTER_API(ServerInfoGetFieldDouble); + REGISTER_API(GetClientInfoById); + REGISTER_API(PublishMessage); + REGISTER_API(SubscribeToServerEvent); + REGISTER_API(SetLRU); + REGISTER_API(GetLRU); + REGISTER_API(SetLFU); + REGISTER_API(GetLFU); + REGISTER_API(BlockClientOnKeys); + REGISTER_API(SignalKeyAsReady); + REGISTER_API(GetBlockedClientReadyKey); + REGISTER_API(GetUsedMemoryRatio); + REGISTER_API(MallocSize); + REGISTER_API(ScanCursorCreate); + REGISTER_API(ScanCursorDestroy); + REGISTER_API(ScanCursorRestart); + REGISTER_API(Scan); + REGISTER_API(ScanKey); + REGISTER_API(CreateModuleUser); + REGISTER_API(SetModuleUserACL); + REGISTER_API(FreeModuleUser); + REGISTER_API(DeauthenticateAndCloseClient); + REGISTER_API(AuthenticateClientWithACLUser); + REGISTER_API(AuthenticateClientWithUser); } diff --git a/src/modules/Makefile b/src/modules/Makefile index 4f6b50f2e..5e012d6f1 100644 --- a/src/modules/Makefile +++ b/src/modules/Makefile @@ -13,7 +13,7 @@ endif .SUFFIXES: .c .so .xo .o -all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so +all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so .c.xo: $(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ @@ -46,6 +46,17 @@ hellotimer.so: hellotimer.xo hellodict.xo: ../redismodule.h hellodict.so: hellodict.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +hellohook.xo: ../redismodule.h + +hellohook.so: hellohook.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +helloacl.xo: ../redismodule.h + +helloacl.so: helloacl.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc testmodule.xo: ../redismodule.h diff --git a/src/modules/helloacl.c b/src/modules/helloacl.c new file mode 100644 index 000000000..6766c0a58 --- /dev/null +++ b/src/modules/helloacl.c @@ -0,0 +1,191 @@ +/* ACL API example - An example for performing custom synchronous and + * asynchronous password authentication. + * + * ----------------------------------------------------------------------------- + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "../redismodule.h" +#include +#include + +// A simple global user +static RedisModuleUser *global; +static uint64_t global_auth_client_id = 0; + +/* HELLOACL.REVOKE + * Synchronously revoke access from a user. */ +int RevokeCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (global_auth_client_id) { + RedisModule_DeauthenticateAndCloseClient(ctx, global_auth_client_id); + return RedisModule_ReplyWithSimpleString(ctx, "OK"); + } else { + return RedisModule_ReplyWithError(ctx, "Global user currently not used"); + } +} + +/* HELLOACL.RESET + * Synchronously delete and re-create a module user. */ +int ResetCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModule_FreeModuleUser(global); + global = RedisModule_CreateModuleUser("global"); + RedisModule_SetModuleUserACL(global, "allcommands"); + RedisModule_SetModuleUserACL(global, "allkeys"); + RedisModule_SetModuleUserACL(global, "on"); + + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} + +/* Callback handler for user changes, use this to notify a module of + * changes to users authenticated by the module */ +void HelloACL_UserChanged(uint64_t client_id, void *privdata) { + REDISMODULE_NOT_USED(privdata); + REDISMODULE_NOT_USED(client_id); + global_auth_client_id = 0; +} + +/* HELLOACL.AUTHGLOBAL + * Synchronously assigns a module user to the current context. */ +int AuthGlobalCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (global_auth_client_id) { + return RedisModule_ReplyWithError(ctx, "Global user currently used"); + } + + RedisModule_AuthenticateClientWithUser(ctx, global, HelloACL_UserChanged, NULL, &global_auth_client_id); + + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} + +#define TIMEOUT_TIME 1000 + +/* Reply callback for auth command HELLOACL.AUTHASYNC */ +int HelloACL_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + size_t length; + + RedisModuleString *user_string = RedisModule_GetBlockedClientPrivateData(ctx); + const char *name = RedisModule_StringPtrLen(user_string, &length); + + if (RedisModule_AuthenticateClientWithACLUser(ctx, name, length, NULL, NULL, NULL) == + REDISMODULE_ERR) { + return RedisModule_ReplyWithError(ctx, "Invalid Username or password"); + } + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} + +/* Timeout callback for auth command HELLOACL.AUTHASYNC */ +int HelloACL_Timeout(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx, "Request timedout"); +} + +/* Private data frees data for HELLOACL.AUTHASYNC command. */ +void HelloACL_FreeData(RedisModuleCtx *ctx, void *privdata) { + REDISMODULE_NOT_USED(ctx); + RedisModule_FreeString(NULL, privdata); +} + +/* Background authentication can happen here. */ +void *HelloACL_ThreadMain(void *args) { + void **targs = args; + RedisModuleBlockedClient *bc = targs[0]; + RedisModuleString *user = targs[1]; + RedisModule_Free(targs); + + RedisModule_UnblockClient(bc,user); + return NULL; +} + +/* HELLOACL.AUTHASYNC + * Asynchronously assigns an ACL user to the current context. */ +int AuthAsyncCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 2) return RedisModule_WrongArity(ctx); + + pthread_t tid; + RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx, HelloACL_Reply, HelloACL_Timeout, HelloACL_FreeData, TIMEOUT_TIME); + + + void **targs = RedisModule_Alloc(sizeof(void*)*2); + targs[0] = bc; + targs[1] = RedisModule_CreateStringFromString(NULL, argv[1]); + + if (pthread_create(&tid, NULL, HelloACL_ThreadMain, targs) != 0) { + RedisModule_AbortBlock(bc); + return RedisModule_ReplyWithError(ctx, "-ERR Can't start thread"); + } + + return REDISMODULE_OK; +} + +/* This function must be present on each Redis module. It is used in order to + * register the commands into the Redis server. */ +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"helloacl",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.reset", + ResetCommand_RedisCommand,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.revoke", + RevokeCommand_RedisCommand,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.authglobal", + AuthGlobalCommand_RedisCommand,"no-auth",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"helloacl.authasync", + AuthAsyncCommand_RedisCommand,"no-auth",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + global = RedisModule_CreateModuleUser("global"); + RedisModule_SetModuleUserACL(global, "allcommands"); + RedisModule_SetModuleUserACL(global, "allkeys"); + RedisModule_SetModuleUserACL(global, "on"); + + global_auth_client_id = 0; + + return REDISMODULE_OK; +} diff --git a/src/modules/hellohook.c b/src/modules/hellohook.c new file mode 100644 index 000000000..7ab78ed07 --- /dev/null +++ b/src/modules/hellohook.c @@ -0,0 +1,93 @@ +/* Server hooks API example + * + * ----------------------------------------------------------------------------- + * + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "../redismodule.h" +#include +#include +#include +#include + +/* Client state change callback. */ +void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + + RedisModuleClientInfo *ci = data; + printf("Client %s event for client #%llu %s:%d\n", + (sub == REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED) ? + "connection" : "disconnection", + ci->id,ci->addr,ci->port); +} + +void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + + RedisModuleFlushInfo *fi = data; + if (sub == REDISMODULE_SUBEVENT_FLUSHDB_START) { + if (fi->dbnum != -1) { + RedisModuleCallReply *reply; + reply = RedisModule_Call(ctx,"DBSIZE",""); + long long numkeys = RedisModule_CallReplyInteger(reply); + printf("FLUSHDB event of database %d started (%lld keys in DB)\n", + fi->dbnum, numkeys); + RedisModule_FreeCallReply(reply); + } else { + printf("FLUSHALL event started\n"); + } + } else { + if (fi->dbnum != -1) { + printf("FLUSHDB event of database %d ended\n",fi->dbnum); + } else { + printf("FLUSHALL event ended\n"); + } + } +} + +/* This function must be present on each Redis module. It is used in order to + * register the commands into the Redis server. */ +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"hellohook",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_ClientChange, clientChangeCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_FlushDB, flushdbCallback); + return REDISMODULE_OK; +} diff --git a/src/modules/hellotype.c b/src/modules/hellotype.c index ba634c4a1..4f2d1d730 100644 --- a/src/modules/hellotype.c +++ b/src/modules/hellotype.c @@ -129,6 +129,7 @@ int HelloTypeInsert_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, /* Insert the new element. */ HelloTypeInsert(hto,value); + RedisModule_SignalKeyAsReady(ctx,argv[1]); RedisModule_ReplyWithLongLong(ctx,hto->len); RedisModule_ReplicateVerbatim(ctx); @@ -190,6 +191,77 @@ int HelloTypeLen_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int return REDISMODULE_OK; } +/* ====================== Example of a blocking command ==================== */ + +/* Reply callback for blocking command HELLOTYPE.BRANGE, this will get + * called when the key we blocked for is ready: we need to check if we + * can really serve the client, and reply OK or ERR accordingly. */ +int HelloBlock_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); + RedisModuleKey *key = RedisModule_OpenKey(ctx,keyname,REDISMODULE_READ); + int type = RedisModule_KeyType(key); + if (type != REDISMODULE_KEYTYPE_MODULE || + RedisModule_ModuleTypeGetType(key) != HelloType) + { + RedisModule_CloseKey(key); + return REDISMODULE_ERR; + } + + /* In case the key is able to serve our blocked client, let's directly + * use our original command implementation to make this example simpler. */ + RedisModule_CloseKey(key); + return HelloTypeRange_RedisCommand(ctx,argv,argc-1); +} + +/* Timeout callback for blocking command HELLOTYPE.BRANGE */ +int HelloBlock_Timeout(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx,"Request timedout"); +} + +/* Private data freeing callback for HELLOTYPE.BRANGE command. */ +void HelloBlock_FreeData(RedisModuleCtx *ctx, void *privdata) { + REDISMODULE_NOT_USED(ctx); + RedisModule_Free(privdata); +} + +/* HELLOTYPE.BRANGE key first count timeout -- This is a blocking verison of + * the RANGE operation, in order to show how to use the API + * RedisModule_BlockClientOnKeys(). */ +int HelloTypeBRange_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 5) return RedisModule_WrongArity(ctx); + RedisModule_AutoMemory(ctx); /* Use automatic memory management. */ + RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1], + REDISMODULE_READ|REDISMODULE_WRITE); + int type = RedisModule_KeyType(key); + if (type != REDISMODULE_KEYTYPE_EMPTY && + RedisModule_ModuleTypeGetType(key) != HelloType) + { + return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE); + } + + /* Parse the timeout before even trying to serve the client synchronously, + * so that we always fail ASAP on syntax errors. */ + long long timeout; + if (RedisModule_StringToLongLong(argv[4],&timeout) != REDISMODULE_OK) { + return RedisModule_ReplyWithError(ctx, + "ERR invalid timeout parameter"); + } + + /* Can we serve the reply synchronously? */ + if (type != REDISMODULE_KEYTYPE_EMPTY) { + return HelloTypeRange_RedisCommand(ctx,argv,argc-1); + } + + /* Otherwise let's block on the key. */ + void *privdata = RedisModule_Alloc(100); + RedisModule_BlockClientOnKeys(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout,argv+1,1,privdata); + return REDISMODULE_OK; +} /* ========================== "hellotype" type methods ======================= */ @@ -282,5 +354,9 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) HelloTypeLen_RedisCommand,"readonly",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"hellotype.brange", + HelloTypeBRange_RedisCommand,"readonly",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } diff --git a/src/multi.cpp b/src/multi.cpp index 2018e08d9..b4ef7c538 100644 --- a/src/multi.cpp +++ b/src/multi.cpp @@ -108,11 +108,13 @@ void discardCommand(client *c) { /* Send a MULTI command to all the slaves and AOF file. Check the execCommand * implementation for more information. */ void execCommandPropagateMulti(client *c) { - robj *multistring = createStringObject("MULTI",5); - - propagate(cserver.multiCommand,c->db->id,&multistring,1, + propagate(cserver.multiCommand,c->db->id,&shared.multi,1, + PROPAGATE_AOF|PROPAGATE_REPL); +} + +void execCommandPropagateExec(client *c) { + propagate(cserver.execCommand,c->db->id,&shared.exec,1, PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(multistring); } void execCommand(client *c) { @@ -177,7 +179,19 @@ void execCommand(client *c) { must_propagate = 1; } - call(c,g_pserver->loading ? CMD_CALL_NONE : CMD_CALL_FULL); + int acl_retval = ACLCheckCommandPerm(c); + if (acl_retval != ACL_OK) { + addReplyErrorFormat(c, + "-NOPERM ACLs rules changed between the moment the " + "transaction was accumulated and the EXEC call. " + "This command is no longer allowed for the " + "following reason: %s", + (acl_retval == ACL_DENIED_CMD) ? + "no permission to execute the command or subcommand" : + "no permission to touch the specified keys"); + } else { + call(c,g_pserver->loading ? CMD_CALL_NONE : CMD_CALL_FULL); + } /* Commands may alter argc/argv, restore mstate. */ c->mstate.commands[j].argc = c->argc; diff --git a/src/networking.cpp b/src/networking.cpp index 97744c410..8624c4e2c 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -85,32 +85,28 @@ void linkClient(client *c) { * this way removing the client in unlinkClient() will not require * a linear scan, but just a constant time operation. */ c->client_list_node = listLast(g_pserver->clients); - if (c->fd != -1) atomicIncr(g_pserver->rgthreadvar[c->iel].cclients, 1); + if (c->conn != nullptr) atomicIncr(g_pserver->rgthreadvar[c->iel].cclients, 1); uint64_t id = htonu64(c->id); raxInsert(g_pserver->clients_index,(unsigned char*)&id,sizeof(id),c,NULL); } -client *createClient(int fd, int iel) { +client *createClient(connection *conn, int iel) { client *c = (client*)zmalloc(sizeof(client), MALLOC_LOCAL); + serverAssert(conn == nullptr || (iel == (serverTL - g_pserver->rgthreadvar))); c->iel = iel; - /* passing -1 as fd it is possible to create a non connected client. + /* passing NULL as conn it is possible to create a non connected client. * This is useful since all the commands needs to be executed * in the context of a client. When commands are executed in other * contexts (for instance a Lua script) we need a non connected client. */ - if (fd != -1) { + if (conn) { serverAssert(iel == (serverTL - g_pserver->rgthreadvar)); - anetNonBlock(NULL,fd); - anetEnableTcpNoDelay(NULL,fd); + connNonBlock(conn); + connEnableTcpNoDelay(conn); if (cserver.tcpkeepalive) - anetKeepAlive(NULL,fd,cserver.tcpkeepalive); - if (aeCreateFileEvent(g_pserver->rgthreadvar[iel].el,fd,AE_READABLE|AE_READ_THREADSAFE, - readQueryFromClient, c) == AE_ERR) - { - close(fd); - zfree(c); - return NULL; - } + connKeepAlive(conn,cserver.tcpkeepalive); + connSetReadHandler(conn, readQueryFromClient, true); + connSetPrivateData(conn, c); } selectDb(c,0); @@ -120,7 +116,7 @@ client *createClient(int fd, int iel) { fastlock_init(&c->lock, "client"); c->id = client_id; c->resp = 2; - c->fd = fd; + c->conn = conn; c->name = NULL; c->bufpos = 0; c->qb_pos = 0; @@ -179,9 +175,12 @@ client *createClient(int fd, int iel) { c->casyncOpsPending = 0; memset(c->uuid, 0, UUID_BINARY_LEN); + c->auth_callback = NULL; + c->auth_callback_privdata = NULL; + c->auth_module = NULL; listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetMatchMethod(c->pubsub_patterns,listMatchObjects); - if (fd != -1) linkClient(c); + if (conn) linkClient(c); initClientMultiState(c); AssertCorrectThread(c); return c; @@ -250,7 +249,7 @@ int prepareClientToWrite(client *c, bool fAsync) { fAsync = fAsync && !FCorrectThread(c); // Not async if we're on the right thread serverAssert(FCorrectThread(c) || fAsync); if (FCorrectThread(c)) { - serverAssert(c->fd <= 0 || c->lock.fOwnLock()); + serverAssert(c->conn == nullptr || c->lock.fOwnLock()); } else { serverAssert(GlobalLocksAcquired()); } @@ -270,7 +269,7 @@ int prepareClientToWrite(client *c, bool fAsync) { if ((c->flags & CLIENT_MASTER) && !(c->flags & CLIENT_MASTER_FORCE_REPLY)) return C_ERR; - if (c->fd <= 0) return C_ERR; /* Fake client for AOF loading. */ + if (!c->conn) return C_ERR; /* Fake client for AOF loading. */ /* Schedule the client to write the output buffers to the socket, unless * it should already be setup to do so (it has already pending data). */ @@ -636,7 +635,7 @@ void addReplyDoubleCore(client *c, double d, bool fAsync) { if (c->resp == 2) { addReplyBulkCStringCore(c, d > 0 ? "inf" : "-inf", fAsync); } else { - addReplyProtoCore(c, d > 0 ? ",inf\r\n" : "-inf\r\n", + addReplyProtoCore(c, d > 0 ? ",inf\r\n" : ",-inf\r\n", d > 0 ? 6 : 7, fAsync); } } else { @@ -672,7 +671,7 @@ void addReplyHumanLongDouble(client *c, long double d) { decrRefCount(o); } else { char buf[MAX_LONG_DOUBLE_CHARS]; - int len = ld2string(buf,sizeof(buf),d,1); + int len = ld2string(buf,sizeof(buf),d,LD_STR_HUMAN); addReplyProto(c,",",1); addReplyProto(c,buf,len); addReplyProto(c,"\r\n",2); @@ -816,14 +815,25 @@ void addReplyBool(client *c, int b) { * RESP2 had it, so API-wise we have this call, that will emit the correct * RESP2 protocol, however for RESP3 the reply will always be just the * Null type "_\r\n". */ -void addReplyNullArray(client *c) { +void addReplyNullArrayCore(client *c, bool fAsync) +{ if (c->resp == 2) { - addReplyProto(c,"*-1\r\n",5); + addReplyProtoCore(c,"*-1\r\n",5,fAsync); } else { - addReplyProto(c,"_\r\n",3); + addReplyProtoCore(c,"_\r\n",3,fAsync); } } +void addReplyNullArray(client *c) +{ + addReplyNullArrayCore(c, false); +} + +void addReplyNullArrayAsync(client *c) +{ + addReplyNullArrayCore(c, true); +} + /* Create the length prefix of a bulk reply, example: $2234 */ void addReplyBulkLenCore(client *c, robj_roptr obj, bool fAsync) { size_t len = stringObjectLen(obj); @@ -1003,44 +1013,21 @@ int clientHasPendingReplies(client *c) { return (c->bufpos || listLength(c->reply)) && !(c->flags & CLIENT_CLOSE_ASAP); } -#define MAX_ACCEPTS_PER_CALL 1000 -static void acceptCommonHandler(int fd, int flags, char *ip, int iel) { - client *c; - if ((c = createClient(fd, iel)) == NULL) { +void clientAcceptHandler(connection *conn) { + client *c = (client*)connGetPrivateData(conn); + + if (connGetState(conn) != CONN_STATE_CONNECTED) { serverLog(LL_WARNING, - "Error registering fd event for the new client: %s (fd=%d)", - strerror(errno),fd); - return; - } - -#ifdef HAVE_SO_INCOMING_CPU - // Set thread affinity - if (cserver.fThreadAffinity) - { - int cpu = iel; - if (setsockopt(fd, SOL_SOCKET, SO_INCOMING_CPU, &cpu, sizeof(iel)) != 0) - { - serverLog(LL_WARNING, "Failed to set socket affinity"); - } - } -#endif - - /* If maxclient directive is set and this is one client more... close the - * connection. Note that we create the client instead to check before - * for this condition, since now the socket is already set in non-blocking - * mode and we can send an error for free using the Kernel I/O */ - if (listLength(g_pserver->clients) > g_pserver->maxclients) { - const char *err = "-ERR max number of clients reached\r\n"; - - /* That's a best effort error message, don't check write errors */ - if (write(c->fd,err,strlen(err)) == -1) { - /* Nothing to do, Just to avoid the warning... */ - } - g_pserver->stat_rejected_conn++; + "Error accepting a client connection: %s", + connGetLastError(conn)); freeClient(c); return; } + // Set thread affinity + if (cserver.fThreadAffinity) + connSetThreadAffinity(conn, c->iel); + /* If the server is running in protected mode (the default) and there * is no password set, nor a specific interface is bound, we don't accept * requests from non loopback interfaces. Instead we try to explain the @@ -1048,10 +1035,12 @@ static void acceptCommonHandler(int fd, int flags, char *ip, int iel) { if (g_pserver->protected_mode && g_pserver->bindaddr_count == 0 && DefaultUser->flags & USER_FLAG_NOPASS && - !(flags & CLIENT_UNIX_SOCKET) && - ip != NULL) + !(c->flags & CLIENT_UNIX_SOCKET)) { - if (strcmp(ip,"127.0.0.1") && strcmp(ip,"::1")) { + char cip[NET_IP_STR_LEN+1] = { 0 }; + connPeerToString(conn, cip, sizeof(cip)-1, NULL); + + if (strcmp(cip,"127.0.0.1") && strcmp(cip,"::1")) { const char *err = "-DENIED Redis is running in protected mode because protected " "mode is enabled, no bind address was specified, no " @@ -1073,7 +1062,7 @@ static void acceptCommonHandler(int fd, int flags, char *ip, int iel) { "4) Setup a bind address or an authentication password. " "NOTE: You only need to do one of the above things in order for " "the server to start accepting connections from the outside.\r\n"; - if (write(c->fd,err,strlen(err)) == -1) { + if (connWrite(c->conn,err,strlen(err)) == -1) { /* Nothing to do, Just to avoid the warning... */ } g_pserver->stat_rejected_conn++; @@ -1083,7 +1072,67 @@ static void acceptCommonHandler(int fd, int flags, char *ip, int iel) { } g_pserver->stat_numconnections++; + moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED, + c); +} + +#define MAX_ACCEPTS_PER_CALL 1000 +static void acceptCommonHandler(connection *conn, int flags, char *ip, int iel) { + client *c; + UNUSED(ip); + AeLocker locker; + locker.arm(nullptr); + + /* Admission control will happen before a client is created and connAccept() + * called, because we don't want to even start transport-level negotiation + * if rejected. + */ + if (listLength(g_pserver->clients) >= g_pserver->maxclients) { + const char *err = "-ERR max number of clients reached\r\n"; + + /* That's a best effort error message, don't check write errors. + * Note that for TLS connections, no handshake was done yet so nothing is written + * and the connection will just drop. + */ + if (connWrite(conn,err,strlen(err)) == -1) { + /* Nothing to do, Just to avoid the warning... */ + } + g_pserver->stat_rejected_conn++; + connClose(conn); + return; + } + + /* Create connection and client */ + if ((c = createClient(conn, iel)) == NULL) { + char conninfo[100]; + serverLog(LL_WARNING, + "Error registering fd event for the new client: %s (conn: %s)", + connGetLastError(conn), + connGetInfo(conn, conninfo, sizeof(conninfo))); + connClose(conn); /* May be already closed, just ignore errors */ + return; + } + + /* Last chance to keep flags */ c->flags |= flags; + + /* Initiate accept. + * + * Note that connAccept() is free to do two things here: + * 1. Call clientAcceptHandler() immediately; + * 2. Schedule a future call to clientAcceptHandler(). + * + * Because of that, we must do nothing else afterwards. + */ + if (connAccept(conn, clientAcceptHandler) == C_ERR) { + char conninfo[100]; + serverLog(LL_WARNING, + "Error accepting a client connection: %s (conn: %s)", + connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); + freeClient((client*)connGetPrivateData(conn)); + return; + } } void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { @@ -1108,7 +1157,7 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { // We always accept on the same thread LLocalThread: aeAcquireLock(); - acceptCommonHandler(cfd,0,cip, ielCur); + acceptCommonHandler(connCreateAcceptedSocket(cfd),0,cip,ielCur); aeReleaseLock(); } else @@ -1123,19 +1172,43 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { char *szT = (char*)zmalloc(NET_IP_STR_LEN, MALLOC_LOCAL); memcpy(szT, cip, NET_IP_STR_LEN); aePostFunction(g_pserver->rgthreadvar[iel].el, [cfd, iel, szT]{ - acceptCommonHandler(cfd,0,szT, iel); + acceptCommonHandler(connCreateAcceptedSocket(cfd),0,szT,iel); zfree(szT); }); } } } +void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) { + int cport, cfd, max = MAX_ACCEPTS_PER_CALL; + char cip[NET_IP_STR_LEN]; + UNUSED(el); + UNUSED(mask); + UNUSED(privdata); + + int ielCur = ielFromEventLoop(el); + while(max--) { + cfd = anetTcpAccept(serverTL->neterr, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno != EWOULDBLOCK) + serverLog(LL_WARNING, + "Accepting client connection: %s", serverTL->neterr); + return; + } + serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); + aeAcquireLock(); + acceptCommonHandler(connCreateAcceptedTLS(cfd, g_pserver->tls_auth_clients),0,cip,ielCur); + aeReleaseLock(); + } +} + void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) { int cfd, max = MAX_ACCEPTS_PER_CALL; UNUSED(el); UNUSED(mask); UNUSED(privdata); + int iel = ielFromEventLoop(el); while(max--) { cfd = anetUnixAccept(serverTL->neterr, fd); if (cfd == ANET_ERR) { @@ -1144,23 +1217,8 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Accepting client connection: %s", serverTL->neterr); return; } - int ielCur = ielFromEventLoop(el); serverLog(LL_VERBOSE,"Accepted connection to %s", g_pserver->unixsocket); - - aeAcquireLock(); - int ielTarget = rand() % cserver.cthreads; - if (ielTarget == ielCur) - { - acceptCommonHandler(cfd,CLIENT_UNIX_SOCKET,NULL, ielCur); - } - else - { - aePostFunction(g_pserver->rgthreadvar[ielTarget].el, [cfd, ielTarget]{ - acceptCommonHandler(cfd,CLIENT_UNIX_SOCKET,NULL, ielTarget); - }); - } - aeReleaseLock(); - + acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL,iel); } } @@ -1199,16 +1257,16 @@ void disconnectSlaves(void) { void unlinkClient(client *c) { listNode *ln; AssertCorrectThread(c); - serverAssert(c->fd == -1 || GlobalLocksAcquired()); - serverAssert(c->fd == -1 || c->lock.fOwnLock()); + serverAssert(c->conn == nullptr || GlobalLocksAcquired()); + serverAssert(c->conn == nullptr || c->lock.fOwnLock()); /* If this is marked as current client unset it. */ if (serverTL && serverTL->current_client == c) serverTL->current_client = NULL; - /* Certain operations must be done only if the client has an active socket. + /* Certain operations must be done only if the client has an active connection. * If the client was already unlinked or if it's a "fake client" the - * fd is already set to -1. */ - if (c->fd != -1) { + * conn is already set to NULL. */ + if (c->conn) { /* Remove from the list of active clients. */ if (c->client_list_node) { uint64_t id = htonu64(c->id); @@ -1217,22 +1275,23 @@ void unlinkClient(client *c) { c->client_list_node = NULL; } - /* In the case of diskless replication the fork is writing to the - * sockets and just closing the fd isn't enough, if we don't also - * shutdown the socket the fork will continue to write to the replica - * and the salve will only find out that it was disconnected when - * it will finish reading the rdb. */ - if ((c->flags & CLIENT_SLAVE) && - (c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)) { - shutdown(c->fd, SHUT_RDWR); + /* Check if this is a replica waiting for diskless replication (rdb pipe), + * in which case it needs to be cleaned from that list */ + if (c->flags & CLIENT_SLAVE && + c->replstate == SLAVE_STATE_WAIT_BGSAVE_END && + g_pserver->rdb_pipe_conns) + { + int i; + for (i=0; i < g_pserver->rdb_pipe_numconns; i++) { + if (g_pserver->rdb_pipe_conns[i] == c->conn) { + rdbPipeWriteHandlerConnRemoved(c->conn); + g_pserver->rdb_pipe_conns[i] = NULL; + break; + } + } } - - /* Unregister async I/O handlers and close the socket. */ - aeDeleteFileEvent(g_pserver->rgthreadvar[c->iel].el,c->fd,AE_READABLE); - aeDeleteFileEvent(g_pserver->rgthreadvar[c->iel].el,c->fd,AE_WRITABLE); - close(c->fd); - c->fd = -1; - + connClose(c->conn); + c->conn = NULL; atomicDecr(g_pserver->rgthreadvar[c->iel].cclients, 1); } @@ -1277,7 +1336,7 @@ void unlinkClient(client *c) { bool freeClient(client *c) { listNode *ln; - serverAssert(c->fd == -1 || GlobalLocksAcquired()); + serverAssert(c->conn == nullptr || GlobalLocksAcquired()); AssertCorrectThread(c); std::unique_locklock)> ulock(c->lock); @@ -1288,6 +1347,16 @@ bool freeClient(client *c) { return false; } + /* For connected clients, call the disconnection event of modules hooks. */ + if (c->conn) { + moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, + c); + } + + /* Notify module system that this client auth status changed. */ + moduleNotifyUserChanged(c); + /* If it is our master that's beging disconnected we should make sure * to cache the state to try a partial resynchronization later. * @@ -1355,6 +1424,11 @@ bool freeClient(client *c) { if (c->flags & CLIENT_SLAVE && listLength(g_pserver->slaves) == 0) g_pserver->repl_no_slaves_since = g_pserver->unixtime; refreshGoodSlavesCount(); + /* Fire the replica change modules event. */ + if (c->replstate == SLAVE_STATE_ONLINE) + moduleFireServerEvent(REDISMODULE_EVENT_REPLICA_CHANGE, + REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE, + NULL); } /* Master/replica cleanup Case 2: @@ -1437,12 +1511,13 @@ client *lookupClientByID(uint64_t id) { /* Write data in output buffers to client. Return C_OK if the client * is still valid after the call, C_ERR if it was freed because of some - * error. + * error. If handler_installed is set, it will attempt to clear the + * write event. * * This function is called by threads, but always with handler_installed * set to 0. So when handler_installed is set to 0 the function must be * thread safe. */ -int writeToClient(int fd, client *c, int handler_installed) { +int writeToClient(client *c, int handler_installed) { ssize_t nwritten = 0, totwritten = 0; clientReplyBlock *o; AssertCorrectThread(c); @@ -1451,8 +1526,7 @@ int writeToClient(int fd, client *c, int handler_installed) { while(clientHasPendingReplies(c)) { if (c->bufpos > 0) { - nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); - + nwritten = connWrite(c->conn,c->buf+c->sentlen,c->bufpos-c->sentlen); if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -1471,10 +1545,8 @@ int writeToClient(int fd, client *c, int handler_installed) { continue; } - nwritten = write(fd, o->buf() + c->sentlen, o->used - c->sentlen); - if (nwritten <= 0) - break; - + nwritten = connWrite(c->conn, o->buf() + c->sentlen, o->used - c->sentlen); + if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -1509,11 +1581,11 @@ int writeToClient(int fd, client *c, int handler_installed) { g_pserver->stat_net_output_bytes += totwritten; if (nwritten == -1) { - if (errno == EAGAIN) { + if (connGetState(c->conn) == CONN_STATE_CONNECTED) { nwritten = 0; } else { serverLog(LL_VERBOSE, - "Error writing to client: %s", strerror(errno)); + "Error writing to client: %s", connGetLastError(c->conn)); freeClientAsync(c); return C_ERR; @@ -1528,7 +1600,7 @@ int writeToClient(int fd, client *c, int handler_installed) { } if (!clientHasPendingReplies(c)) { c->sentlen = 0; - if (handler_installed) aeDeleteFileEvent(g_pserver->rgthreadvar[c->iel].el,c->fd,AE_WRITABLE); + if (handler_installed) connSetWriteHandler(c->conn, NULL); /* Close connection after entire reply has been sent. */ if (c->flags & CLIENT_CLOSE_AFTER_REPLY) { @@ -1540,12 +1612,9 @@ int writeToClient(int fd, client *c, int handler_installed) { } /* Write event handler. Just send data to the client. */ -void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { - UNUSED(mask); - client *c = (client*)privdata; - - serverAssert(ielFromEventLoop(el) == c->iel); - if (writeToClient(fd,c,1) == C_ERR) +void sendReplyToClient(connection *conn) { + client *c = (client*)connGetPrivateData(conn); + if (writeToClient(c,1) == C_ERR) { AeLocker ae; c->lock.lock(); @@ -1633,7 +1702,7 @@ void ProcessPendingAsyncWrites() std::lock_guardlock)> lock(c->lock); serverAssert(c->casyncOpsPending > 0); c->casyncOpsPending--; - aeCreateFileEvent(g_pserver->rgthreadvar[c->iel].el, c->fd, AE_WRITABLE|AE_WRITE_THREADSAFE, sendReplyToClient, c); + connSetWriteHandler(c->conn, sendReplyToClient, true); }, false) == AE_ERR ) { @@ -1682,7 +1751,7 @@ int handleClientsWithPendingWrites(int iel) { std::unique_locklock)> lock(c->lock); /* Try to write buffers to the client socket. */ - if (writeToClient(c->fd,c,0) == C_ERR) + if (writeToClient(c,0) == C_ERR) { if (c->flags & CLIENT_CLOSE_ASAP) { @@ -1698,7 +1767,7 @@ int handleClientsWithPendingWrites(int iel) { /* If after the synchronous writes above we still have data to * output to the client, we need to install the writable handler. */ if (clientHasPendingReplies(c)) { - if (aeCreateFileEvent(g_pserver->rgthreadvar[c->iel].el, c->fd, ae_flags, sendReplyToClient, c) == AE_ERR) + if (connSetWriteHandlerWithBarrier(c->conn, sendReplyToClient, ae_flags, true) == C_ERR) freeClientAsync(c); } } @@ -1753,8 +1822,8 @@ void resetClient(client *c) { void protectClient(client *c) { c->flags |= CLIENT_PROTECTED; AssertCorrectThread(c); - aeDeleteFileEvent(g_pserver->rgthreadvar[c->iel].el,c->fd,AE_READABLE); - aeDeleteFileEvent(g_pserver->rgthreadvar[c->iel].el,c->fd,AE_WRITABLE); + connSetReadHandler(c->conn,NULL); + connSetWriteHandler(c->conn,NULL); } /* This will undo the client protection done by protectClient() */ @@ -1762,7 +1831,7 @@ void unprotectClient(client *c) { AssertCorrectThread(c); if (c->flags & CLIENT_PROTECTED) { c->flags &= ~CLIENT_PROTECTED; - aeCreateFileEvent(g_pserver->rgthreadvar[c->iel].el,c->fd,AE_READABLE|AE_READ_THREADSAFE,readQueryFromClient,c); + connSetReadHandler(c->conn,readQueryFromClient, true); if (clientHasPendingReplies(c)) clientInstallWriteHandler(c); } } @@ -1824,12 +1893,8 @@ int processInlineBuffer(client *c) { /* Create redis objects for all arguments. */ for (c->argc = 0, j = 0; j < argc; j++) { - if (sdslen(argv[j])) { - c->argv[c->argc] = createObject(OBJ_STRING,argv[j]); - c->argc++; - } else { - sdsfree(argv[j]); - } + c->argv[c->argc] = createObject(OBJ_STRING,argv[j]); + c->argc++; } sds_free(argv); return C_OK; @@ -2143,14 +2208,13 @@ void processInputBufferAndReplicate(client *c) { } } -void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { - client *c = (client*) privdata; +void readQueryFromClient(connection *conn) { + client *c = (client*)connGetPrivateData(conn); + serverAssert(conn == c->conn); int nread, readlen; size_t qblen; - UNUSED(el); - UNUSED(mask); - serverAssert(mask & AE_READ_THREADSAFE); - serverAssert(c->iel == ielFromEventLoop(el)); + + serverAssert(FCorrectThread(c)); AeLocker aelock; AssertCorrectThread(c); @@ -2178,14 +2242,14 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { qblen = sdslen(c->querybuf); if (c->querybuf_peak < qblen) c->querybuf_peak = qblen; c->querybuf = sdsMakeRoomFor(c->querybuf, readlen); - - nread = read(fd, c->querybuf+qblen, readlen); + + nread = connRead(c->conn, c->querybuf+qblen, readlen); if (nread == -1) { - if (errno == EAGAIN) { + if (connGetState(conn) == CONN_STATE_CONNECTED) { return; } else { - serverLog(LL_VERBOSE, "Reading from client: %s",strerror(errno)); + serverLog(LL_VERBOSE, "Reading from client: %s",connGetLastError(c->conn)); freeClientAsync(c); return; } @@ -2266,7 +2330,7 @@ void genClientPeerId(client *client, char *peerid, snprintf(peerid,peerid_len,"%s:0",g_pserver->unixsocket); } else { /* TCP client. */ - anetFormatPeer(client->fd,peerid,peerid_len); + connFormatPeer(client->conn,peerid,peerid_len); } } @@ -2287,8 +2351,7 @@ char *getClientPeerId(client *c) { /* Concatenate a string representing the state of a client in an human * readable format, into the sds string 's'. */ sds catClientInfoString(sds s, client *client) { - char flags[16], events[3], *p; - int emask; + char flags[16], events[3], conninfo[CONN_INFO_LEN], *p; p = flags; if (client->flags & CLIENT_SLAVE) { @@ -2312,17 +2375,18 @@ sds catClientInfoString(sds s, client *client) { if (p == flags) *p++ = 'N'; *p++ = '\0'; - emask = client->fd == -1 ? 0 : aeGetFileEvents(g_pserver->rgthreadvar[client->iel].el,client->fd); p = events; - if (emask & AE_READABLE) *p++ = 'r'; - if (emask & AE_WRITABLE) *p++ = 'w'; + if (client->conn) { + if (connHasReadHandler(client->conn)) *p++ = 'r'; + if (connHasWriteHandler(client->conn)) *p++ = 'w'; + } *p = '\0'; return sdscatfmt(s, - "id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", + "id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s", (unsigned long long) client->id, getClientPeerId(client), - client->fd, - client->name ? (char*)ptrFromObj(client->name) : "", + connGetInfo(client->conn, conninfo, sizeof(conninfo)), + client->name ? (char*)szFromObj(client->name) : "", (long long)(g_pserver->unixtime - client->ctime), (long long)(g_pserver->unixtime - client->lastinteraction), flags, @@ -2336,7 +2400,8 @@ sds catClientInfoString(sds s, client *client) { (unsigned long long) listLength(client->reply), (unsigned long long) getClientOutputBufferMemoryUsage(client), events, - client->lastcmd ? client->lastcmd->name : "NULL"); + client->lastcmd ? client->lastcmd->name : "NULL", + client->puser ? client->puser->name : "(superuser)"); } sds getAllClientsInfoString(int type) { @@ -2374,7 +2439,6 @@ int clientSetNameOrReply(client *c, robj *name) { if (len == 0) { if (c->name) decrRefCount(c->name); c->name = NULL; - addReply(c,shared.ok); return C_OK; } @@ -2402,20 +2466,21 @@ void clientCommand(client *c) { if (c->argc == 2 && !strcasecmp((const char*)ptrFromObj(c->argv[1]),"help")) { const char *help[] = { -"id -- Return the ID of the current connection.", -"getname -- Return the name of the current connection.", -"kill -- Kill connection made from .", -"kill