diff --git a/.gitignore b/.gitignore index cf904522d..a188cfc82 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ dump.rdb redis-benchmark redis-check-aof +redis-check-rdb redis-check-dump redis-cli redis-sentinel @@ -19,9 +20,11 @@ src/transfer.sh src/configs redis.ds src/redis.conf +src/nodes.conf deps/lua/src/lua deps/lua/src/luac deps/lua/src/liblua.a .make-* .prerequisites *.dSYM +Makefile.dep diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 36317ca35..ce472159e 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,85 +1,16 @@ -Redis 2.6 release notes +Hello! This file is just a placeholder, since this is the "unstable" branch +of Redis, the place where all the development happens. -Migrating from 2.4 to 2.6 -========================= +There is no release notes for this branch, it gets forked into another branch +every time there is a partial feature freeze in order to eventually create +a new stable release. -Redis 2.4 is mostly a strict subset of 2.6. However there are a few things -that you should be aware of: +Usually "unstable" is stable enough for you to use it in development environments +however you should never use it in production environments. It is possible +to download the latest stable release here: -* You can't use .rdb and AOF files generated with 2.6 into a 2.4 instance. -* 2.6 slaves can be attached to 2.4 masters, but not the contrary, and only - for the time needed to perform the version upgrade. + http://download.redis.io/releases/redis-stable.tar.gz -There are also a few API differences, that are unlikely to cause problems, -but it is better to keep them in mind: +More information is available at http://redis.io -* SORT now will refuse to sort in numerical mode elements that can't be parsed - as numbers. -* EXPIREs now all have millisecond resolution (but this is very unlikely to - break code that was not conceived exploting the previous resolution error - in some way.) -* INFO output is a bit different now, and contains empty lines and comments - starting with '#'. All the major clients should be already fixed to work - with the new INFO format. - -Also the following redis.conf and CONFIG GET / SET parameters changed name: - - * hash-max-zipmap-entries, now replaced by hash-max-ziplist-entries - * hash-max-zipmap-value, now replaced by hash-max-ziplist-value - * glueoutputbuf was now completely removed as it does not make sense - ---------- -CHANGELOG ---------- - -What's new in Redis 2.6.0 -========================= - -UPGRADE URGENCY: We suggest new users to start with 2.6.0, and old users to - upgrade after some testing of the application with the new - Redis version. - -* Server side Lua scripting, see http://redis.io/commands/eval -* Virtual Memory removed (was deprecated in 2.4) -* Hardcoded limits about max number of clients removed. -* AOF low level semantics is generally more sane, and especially when used - in slaves. -* Milliseconds resolution expires, also added new commands with milliseconds - precision (PEXPIRE, PTTL, ...). -* Clients max output buffer soft and hard limits. You can specifiy different - limits for different classes of clients (normal,pubsub,slave). -* AOF is now able to rewrite aggregate data types using variadic commands, - often producing an AOF that is faster to save, load, and is smaller in size. -* Every redis.conf directive is now accepted as a command line option for the - redis-server binary, with the same name and number of arguments. -* Hash table seed randomization for protection against collisions attacks. -* Performances improved when writing large objects to Redis. -* Significant parts of the core refactored or rewritten. New internal APIs - and core changes allowed to develop Redis Cluster on top of the new code, - however for 2.6 all the cluster code was removed, and will be released with - Redis 3.0 when it is more complete and stable. -* Redis ASCII art logo added at startup. -* Crash report on memory violation or failed asserts improved significantly - to make debugging of hard to catch bugs simpler. -* redis-benchmark improvements: ability to run selected tests, - CSV output, faster, better help. -* redis-cli improvements: --eval for comfortable development of Lua scripts. -* SHUTDOWN now supports two optional arguments: "SAVE" and "NOSAVE". -* INFO output split into sections, the command is now able to just show - pecific sections. -* New statistics about how many time a command was called, and how much - execution time it used (INFO commandstats). -* More predictable SORT behavior in edge cases. -* INCRBYFLOAT and HINCRBYFLOAT commands. - --------------------------------------------------------------------------------- - -Credits: Where not specified the implementation and design are done by -Salvatore Sanfilippo and Pieter Noordhuis. Thanks to VMware for making all -this possible. Also many thanks to all the other contributors and the amazing -community we have. - -See commit messages for more credits. - -Cheers, -Salvatore +Happy hacking! diff --git a/BUGS b/BUGS index 96d52bf8b..a8e936892 100644 --- a/BUGS +++ b/BUGS @@ -1 +1 @@ -Plese check https://github.com/antirez/redis/issues +Please check https://github.com/antirez/redis/issues diff --git a/CONTRIBUTING b/CONTRIBUTING index 06e1e9660..f57de3fd9 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -12,17 +12,19 @@ each source file that you contribute. PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected bugs in the Github issues system. We'll be very happy to help you and provide - all the support in the Redis Google Group. + all the support at the Reddit sub: - Redis Google Group address: - - https://groups.google.com/forum/?fromgroups#!forum/redis-db + http://reddit.com/r/redis + + There is also an active community of Redis users at Stack Overflow: + + http://stackoverflow.com/questions/tagged/redis # How to provide a patch for a new feature -1. Drop a message to the Redis Google Group with a proposal of semantics/API. +1. If it is a major feature or a semantical change, please post it as a new submission in r/redis on Reddit at http://reddit.com/r/redis. Try to be passionate about why the feature is needed, make users upvote your proposal to gain traction and so forth. Read feedbacks about the community. But in this first step **please don't write code yet**. -2. If in steps 1 you get an acknowledge from the project leaders, use the +2. If in step 1 you get an acknowledgment from the project leaders, use the following procedure to submit a patch: a. Fork Redis on github ( http://help.github.com/fork-a-repo/ ) @@ -31,4 +33,6 @@ each source file that you contribute. d. Initiate a pull request on github ( http://help.github.com/send-pull-requests/ ) e. Done :) +For minor fixes just open a pull request on Github. + Thanks! diff --git a/COPYING b/COPYING index c8665ba67..ac68e012b 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (c) 2006-2012, Salvatore Sanfilippo +Copyright (c) 2006-2015, Salvatore Sanfilippo All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Changelog b/Changelog deleted file mode 100644 index f72746663..000000000 --- a/Changelog +++ /dev/null @@ -1,1032 +0,0 @@ -2010-07-01 gitignore modified (antirez) -2010-06-22 redis.c split into many different C files. (antirez) -2010-06-16 more pub/sub tests (Pieter Noordhuis) -2010-06-15 initial basic pub/sub tests (Pieter Noordhuis) -2010-06-15 fix BLPOP/BRPOP to use the wrapped function for list length (Pieter Noordhuis) -2010-06-15 tests for BLPOP/BRPOP via an option in the tcl client that defers reading the reply (Pieter Noordhuis) -2010-06-14 TODO updated (antirez) -2010-06-14 Merge branch 'ltrim-tests' of git://github.com/pietern/redis (antirez) -2010-06-14 rename "list" to "linkedlist" to be more verbose (Pieter Noordhuis) -2010-06-14 allow running the test suite against an external Redis instance, without auto spawning (antirez) -2010-06-14 change ltrim tests to cover all min/max cases and add stronger stresser (Pieter Noordhuis) -2010-06-13 Fixed deps in makefile and mkreleasehdr.sh script to really take advantage of the new trick to avoid recompilation of redis.c on git sha1 or dirty status change (antirez) -2010-06-13 hopefully faster recompiling with a trick (antirez) -2010-06-13 fixed a bug in rdbLoadObject abount specially encoded objects (antirez) -2010-06-13 use raw strings when loading a hash from the rdb into a zipmap (Pieter Noordhuis) -2010-06-12 Merge branch 'expire' of git://github.com/pietern/redis (antirez) -2010-06-11 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-11 LPUSHX, RPUSHX, LINSERT only work on non-empty lists, so there are no clients waiting for a push (Pieter Noordhuis) -2010-06-11 make LINSERT return -1 when the value could not be inserted (Pieter Noordhuis) -2010-06-11 check if the list encoding needs to be changed on LPUSHX, RPUSHX, LINSERT (Pieter Noordhuis) -2010-06-11 make sure the value to insert is string encoded (Pieter Noordhuis) -2010-06-11 rename vars, move arguments, add comments (Pieter Noordhuis) -2010-06-11 always iterate from head to tail on LINSERT (Pieter Noordhuis) -2010-06-11 use REDIS_TAIL to insert AFTER an entry and REDIS_HEAD to insert BEFORE an entry (Pieter Noordhuis) -2010-06-11 move listTypeInsert to be grouped with other wrapper functions (Pieter Noordhuis) -2010-06-11 squashed merge from robey/twitter3: LINSERT BEFORE|AFTER, LPUSHX, RPUSHX (Pieter Noordhuis) -2010-06-09 remove pop function and the sds dependency; can be implemented using get+delete (Pieter Noordhuis) -2010-06-07 compute swappability for ziplist encoded lists (Pieter Noordhuis) -2010-06-07 reuse the sds from the main dictionary in the expiration dictionary (Pieter Noordhuis) -2010-06-07 TODO updated (antirez) -2010-06-07 encode integers while loading an hash (antirez) -2010-06-05 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-05 fixed two leaks for the dual encoded lists (Pieter Noordhuis) -2010-06-04 TODO updated (antirez) -2010-06-04 DISCSARD now unwatches all keys, as it should (antirez) -2010-06-04 generated tests for different encodings to avoid test code duplication (Pieter Noordhuis) -2010-06-04 refactor list tests to test both encodings; implemented assert functions (Pieter Noordhuis) -2010-06-04 renamed hash wrapper functions to match wrapper function naming convention: "Type" (Pieter Noordhuis) -2010-06-04 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-04 Merge branch 'smallkeys' (antirez) -2010-06-04 safety assert in listTypeNext (Pieter Noordhuis) -2010-06-04 renamed list wrapper functions to be more verbose (Pieter Noordhuis) -2010-06-04 add thresholds for converting a ziplist to a real list (Pieter Noordhuis) -2010-06-04 merge antirez/smallkeys (Pieter Noordhuis) -2010-06-03 test restored (antirez) -2010-06-03 memory leak introduced in the latest big changes fixed (antirez) -2010-06-03 Fixed VM bugs introduced with the top level keys as sds strings changes (antirez) -2010-06-03 top level keys are no longer redis objects but sds strings. There are still a few bugs to fix when VM is enabled (antirez) -2010-06-03 update Makefile to include ziplist.o (Pieter Noordhuis) -2010-06-03 use ziplists in SORT STORE until the thresholds are determined (Pieter Noordhuis) -2010-06-03 Merge branch 'testsuite' of git://github.com/pietern/redis (antirez) -2010-06-03 Merge branch 'testsuite' of git://github.com/pietern/redis into smallkeys (antirez) -2010-06-03 tag memory leak check on kill server as "leaks" (Pieter Noordhuis) -2010-06-03 tag test with sleep() as slow (Pieter Noordhuis) -2010-06-03 make sure the config it returned when called without code (Pieter Noordhuis) -2010-06-03 tag more slow tests (Pieter Noordhuis) -2010-06-03 change how arguments are passed from the AOF tests (Pieter Noordhuis) -2010-06-03 scope res variable outside test (Pieter Noordhuis) -2010-06-02 tags for existing tests (Pieter Noordhuis) -2010-06-02 pass tags to filter and match via arguments (Pieter Noordhuis) -2010-06-02 basic support to tag tests (Pieter Noordhuis) -2010-06-02 changed how server.tcl accepts options to support more directives without requiring more arguments to the proc (Pieter Noordhuis) -2010-06-02 removed obsolete code (Pieter Noordhuis) -2010-06-02 catch exceptions in the server proc, to be able to kill the entire chain of running servers (Pieter Noordhuis) -2010-06-02 Merge branch 'master' into smallkeys (antirez) -2010-06-02 smarter swapout policy on AOF too (antirez) -2010-06-02 better swapout policy while loading RDB file (antirez) -2010-06-02 minor code comment change (antirez) -2010-06-01 use integer types from stdint.h to be more verbose on the size in bytes of encoded elements. update list length to use 2 bytes instead of 1. (Pieter Noordhuis) -2010-06-01 added stress test for heavy i/o in ziplists (Pieter Noordhuis) -2010-06-01 fix signedness errors in ziplist testing code (Pieter Noordhuis) -2010-06-01 minor code movements and free object pull restored to 1 million (antirez) -2010-06-01 TODO updated with syslog plans for 2.2 (antirez) -2010-06-01 Debug message was printing stuff that are sometimes not initialized/valid (antirez) -2010-06-01 Merge branch 'smallkeys' of github.com:antirez/redis into smallkeys (antirez) -2010-06-01 fixed a few comments (antirez) -2010-06-01 fixed bugs introduced in the rewrite of the new VM engine (antirez) -2010-05-31 support rewriting the AOF with dual list encoding (Pieter Noordhuis) -2010-05-31 small refactor of fwrite* commands for AOF rewrite to allow writing a bulk long long (Pieter Noordhuis) -2010-05-31 use list wrapper functions in computing the dataset digest (Pieter Noordhuis) -2010-05-31 ziplistNext should work as expected when called with a pointer to ZIP_END (Pieter Noordhuis) -2010-05-31 update SORT to work with the dual list encoding (Pieter Noordhuis) -2010-05-31 function to create a new ziplist encoded list (Pieter Noordhuis) -2010-05-31 fixed missing incrRefCount (antirez) -2010-05-31 support rdb saving/loading with dual list encoding (Pieter Noordhuis) -2010-05-31 fixed signedness and disambiguate variable names (Pieter Noordhuis) -2010-05-31 added rdb save function to directly save long long values (Pieter Noordhuis) -2010-05-31 update RPOPLPUSH to support dual encoding (Pieter Noordhuis) -2010-05-31 update list iteration semantic to work as expected (i.e. "while(lNext(..))") (Pieter Noordhuis) -2010-05-31 ziplistDelete no longer needs a direction now ziplistPrev is fixed (Pieter Noordhuis) -2010-05-31 ziplistPrev should return the tail when the argument is ZIP_END (Pieter Noordhuis) -2010-05-31 first step of VM rewrite. blocking VM tests passing, more work needed in the async side (antirez) -2010-05-31 Merge branch 'no-appendfsync-on-rewrite' (antirez) -2010-05-30 fix LREM to remove *all* occurances when a zero argument is given (Pieter Noordhuis) -2010-05-30 fixed LINDEX to always return bulk response (Pieter Noordhuis) -2010-05-30 the tail offset must be an integer pointer to hold a 32-bit offset (Pieter Noordhuis) -2010-05-30 update LREM to support dual encoding via extra iteration primitives (Pieter Noordhuis) -2010-05-30 support dual encoding in LTRIM (Pieter Noordhuis) -2010-05-30 update LRANGE to use basic iteration code to support dual encoding (Pieter Noordhuis) -2010-05-30 inline support for dual encoding in the LINDEX and LSET commands (Pieter Noordhuis) -2010-05-30 generic pop and length function for ziplist encoding (Pieter Noordhuis) -2010-05-30 generic push function that supports the dual encoding (Pieter Noordhuis) -2010-05-30 change delete function to accept a direction argument, so "p" can be properly updated (Pieter Noordhuis) -2010-05-30 expose extra functionality from ziplist.c (Pieter Noordhuis) -2010-05-30 code style consistency fixes (Pieter Noordhuis) -2010-05-29 ziplistIndex now accepts negative indices (Pieter Noordhuis) -2010-05-29 fix compile warnings (Pieter Noordhuis) -2010-05-29 use simpler encoding for the length of the previous entry (Pieter Noordhuis) -2010-05-29 replace functions to get pointers to head and tail by macros (Pieter Noordhuis) -2010-05-29 function to insert an element at an arbitrary position in the list (Pieter Noordhuis) -2010-05-29 extract a generic delete function that can be used in pop and delete(range) (Pieter Noordhuis) -2010-05-29 use the entry struct in zipRawEntryLength (Pieter Noordhuis) -2010-05-29 rename argument names to s* to disambiguate from e* (Pieter Noordhuis) -2010-05-29 change ziplistRepr to use the entry struct (Pieter Noordhuis) -2010-05-29 modify compare function to check if the encoding is equal before comparing (Pieter Noordhuis) -2010-05-29 use a struct to retrieve all details for an entry (Pieter Noordhuis) -2010-05-29 initial implementation for making the ziplist doubly linked (Pieter Noordhuis) -2010-05-29 fix some warnings (Pieter Noordhuis) -2010-05-29 add function to retrieve ziplist size in bytes (Pieter Noordhuis) -2010-05-22 fix compare function of ziplist to only load integer from ziplist when it is encoded as integer (Pieter Noordhuis) -2010-05-22 add function to retrieve length of ziplist (Pieter Noordhuis) -2010-05-22 re-introduce ZIP_BIGLEN for clarity (Pieter Noordhuis) -2010-05-22 added header ziplist.h (Pieter Noordhuis) -2010-05-22 code to compare strings with entries in ziplist, regardless of their encoding (Pieter Noordhuis) -2010-05-22 updated iteration code to work well with different encodings (Pieter Noordhuis) -2010-05-22 move code from zip.c to ziplist.c (Pieter Noordhuis) -2010-05-22 partial revert of c80df5 because ziplist functions are starting to divert too much from zipmap functions (Pieter Noordhuis) -2010-05-22 initial work for integer encoding in ziplists (Pieter Noordhuis) -2010-05-22 move length housekeeping to a macro (Pieter Noordhuis) -2010-05-21 allow entries to be deleted in place when iterating over a ziplist (Pieter Noordhuis) -2010-05-21 allow pointer to be stored to current element when iterating over ziplist (Pieter Noordhuis) -2010-05-21 rename ziplistDelete to ziplistDeleteRange (Pieter Noordhuis) -2010-05-21 code to delete an inner range from the ziplist (Pieter Noordhuis) -2010-05-21 check if *value is non-NULL before setting it (Pieter Noordhuis) -2010-05-21 change iteration code to avoid allocating a new sds for each traversed entry (Pieter Noordhuis) -2010-05-21 code to iterate over a ziplist (Pieter Noordhuis) -2010-05-21 implementation for a ziplist with push and pop support (Pieter Noordhuis) -2010-05-21 extracted general methods to zip.c for reuse in other zip* structures (Pieter Noordhuis) -2010-05-28 command table size calculated with sizeof (antirez) -2010-05-28 use qsort and bsearch to lookup commands in O(log(N)) instead of O(N) (Pieter Noordhuis) -2010-05-28 Merge branch 'cli-stdin' of git://github.com/pietern/redis (antirez) -2010-05-28 Fixed ZINCR Nan bugs leading to server crash and added tests (antirez) -2010-05-28 redis.conf new features the new option, a minor typo preventing the compilation fixed (antirez) -2010-05-28 don't fsync after a rewrite if appendfsync is set to no. use aof_fsycn instead of fsync where appropriate (antirez) -2010-05-28 added new option no-appendfsync-on-rewrite to avoid blocking on fsync() in the main thread while a background process is doing big I/O (antirez) -2010-05-28 Added Git sha1 and dirty status in redis-server -v output (antirez) -2010-05-28 changed the message in the Makefile with the new command like to run the test suite (antirez) -2010-05-27 Fixed typo. (Vincent Palmer) -2010-05-27 new multi/exec tests (antirez) -2010-05-26 build command outside while loop (Pieter Noordhuis) -2010-05-26 require the flag "-c" to be used for redis-cli to read the last argument from stdin (Pieter Noordhuis) -2010-05-26 Merge branch 'master' into nested-multi (antirez) -2010-05-26 Fix EXEC bug that was leaving the client in dirty status when used with WATCH (antirez) -2010-05-26 raise error on nested MULTI and WATCH inside multi (antirez) -2010-05-25 allow regular sets to be passed to zunionstore/zinterstore (Pieter Noordhuis) -2010-05-25 Version is now 2.1.1 (antirez) -2010-05-25 RENAME is now WATCH-aware (antirez) -2010-05-25 TODO updated (antirez) -2010-05-25 WATCH is now able to detect keys removed by FLUSHALL and FLUSHDB (antirez) -2010-05-25 WATCH tests (antirez) -2010-05-25 minor bug fixed in WATCH (antirez) -2010-05-25 WATCH for MULTI/EXEC (CAS alike concurrency) (antirez) -2010-05-25 gitignore updated (antirez) -2010-05-21 Master is now already unfreezed, unstable, and ready to hacking sessions! (antirez) -2010-05-21 Merge branch 'solaris' of git://github.com/pietern/redis (antirez) -2010-05-21 Changelog updated (antirez) -2010-05-21 redis version is now 1.3.14 (aka 2.0.0 RC1) (antirez) -2010-05-21 html doc updated (antirez) -2010-05-21 by default test with valgrind does not show full leak info (antirez) -2010-05-21 minor fix for the skiplist code, resulting in a false positive with valgrind, and in general into a useless small allocation (antirez) -2010-05-21 Merge branch 'master' of git@github.com:antirez/redis (antirez) -2010-05-21 tests suite initial support for valgrind, fixed the old test suite until the new one is able to target a specific host/port (antirez) -2010-05-21 include solaris fixes in sha1.c (Pieter Noordhuis) -2010-05-20 Don't exit with error in tests temp file cleanup if there are no files to clean (antirez) -2010-05-20 fix memory leak on 32-bit builds (Pieter Noordhuis) -2010-05-20 Merge branch 'master' of github.com:antirez/redis (antirez) -2010-05-20 Fix for DEBUG DIGEST (antirez) -2010-05-20 Merge branch 'test_vm' of git://github.com/pietern/redis (antirez) -2010-05-20 code to enable running tests with the vm enabled (Pieter Noordhuis) -2010-05-20 minor change to shutdown (antirez) -2010-05-20 shutdown on SIGTERM (antirez) -2010-05-20 Merge http://github.com/ngmoco/redis (antirez) -2010-05-20 fix compile error on solaris (Pieter Noordhuis) -2010-05-20 added regression for zipmap bug (antirez) -2010-05-20 fix lookup of keys with length larger than ZIPMAP_BIGLEN (Pieter Noordhuis) -2010-05-19 TODO updated (antirez) -2010-05-19 initial tests for AOF (and small changes to server.tcl to support these) (Pieter Noordhuis) -2010-05-19 Merge branch 'master' into integration (Pieter Noordhuis) -2010-05-19 Fix for 'CONFIG SET appendonly no' (antirez) -2010-05-19 It's now possible to turn off and on the AOF via CONFIG (antirez) -2010-05-18 git hash 00000000 in reelase.h when git is not found enabled again after some shell scripting fix that is now compatible with most shells (antirez) -2010-05-18 build fixed when simpler shells are used to create release.h (antirez) -2010-05-18 use git diff when generating release.h to check for dirty status (antirez) -2010-05-18 Solaris fixes (antirez) -2010-05-18 html doc rebuild (antirez) -2010-05-18 buliding of release.h moved into an external script. Avoided recompialtion of redis.c if git sha1 is the same as the previous one (antirez) -2010-05-17 create release.h in make process and add this information to INFO listing (Pieter Noordhuis) -2010-05-16 Redis version is now 1.3.12 (antirez) -2010-05-16 redis version is now 1.3.11 (antirez) -2010-05-16 random refactoring and speedups (antirez) -2010-05-16 faster INCR with very little efforts... (antirez) -2010-05-15 print warnings in redis log when a test raises an exception (very likely to be caused by something like a failed assertion) (Pieter Noordhuis) -2010-05-15 Merge branch 'redis-cli-fix' of http://github.com/tizoc/redis (antirez) -2010-05-15 added pid info to the check memory leaks test, so that those tests don't appear to be duplicated (antirez) -2010-05-15 Merge branch 'integration' of git://github.com/pietern/redis (antirez) -2010-05-14 more endianess detection fix for SHA1 (antirez) -2010-05-14 fixed a warning seen with some GCC version under Linux (antirez) -2010-05-14 initial rough integration test for replication (Pieter Noordhuis) -2010-05-14 store entire server object on the stack instead of just the client (Pieter Noordhuis) -2010-05-14 proc to retrieve values from INFO properties (Pieter Noordhuis) -2010-05-14 one more fix for endianess detection (antirez) -2010-05-14 Fixed sha1.c compilation on Linux, due to endianess detection lameness (antirez) -2010-05-14 ZUNION,ZINTER -> ZUNIONSTORE,ZINTERSTORE (antirez) -2010-05-14 minor fixes to the new test suite, html doc updated (antirez) -2010-05-14 wait for redis-server to be settled and ready for connections (Pieter Noordhuis) -2010-05-14 fix cleaning up tmp folder (Pieter Noordhuis) -2010-05-14 update makefile to use the new test suite (Pieter Noordhuis) -2010-05-14 check for memory leaks before killing a server (Pieter Noordhuis) -2010-05-14 extract code to kill a server to a separate proc (Pieter Noordhuis) -2010-05-14 start servers on different ports to prevent conflicts (Pieter Noordhuis) -2010-05-14 use DEBUG DIGEST in new test suite (Pieter Noordhuis) -2010-05-14 split test suite into multiple files; runs redis-server in isolation (Pieter Noordhuis) -2010-05-14 use DEBUG DIGEST in the test instead of a function that was doing a similar work, but in a much slower and buggy way (antirez) -2010-05-14 Don't rely on cliReadReply being able to return on shutdown (Bruno Deferrari) -2010-05-14 If command is a shutdown, ignore errors on reply (Bruno Deferrari) -2010-05-14 DEBUG DIGEST implemented, in order to improve the ability to test persistence and replication consistency (antirez) -2010-05-13 Add SIGTERM shutdown handling. (Ashley Martens) -2010-05-13 makefile deps updated (antirez) -2010-05-13 conflicts resolved (antirez) -2010-05-13 feed SETEX as SET and EXPIREAT to AOF (Pieter Noordhuis) -2010-05-13 very strong speedup in saving time performance when there are many integers in the dataset. Instead of decoding the object before to pass them to the rdbSaveObject layer we check asap if the object is integer encoded and can be written on disk as an integer. (antirez) -2010-05-13 include limits.h otherwise no double precison macros (antirez) -2010-05-13 explicitly checks with ifdefs if our floating point and long long assumptions are verified (antirez) -2010-05-13 Yet another version of the double saving code, with comments explaining what's happening there (antirez) -2010-05-12 added overflow check in the double -> long long conversion trick to avoid integer overflows. I think this was not needed in practical terms, but it is safer (antirez) -2010-05-12 use withscores when performing the dataset digest (antirez) -2010-05-12 If a float can be casted to a long long without rounding loss, we can use the integer conversion function to write the score on disk. This is a seriuous speedup (antirez) -2010-05-12 fixed compilation warnings in the AOF sanity check tool (antirez) -2010-05-12 Merge branch 'vm-speedup' (antirez) -2010-05-11 fix to return error when calling INCR on a non-string type (Pieter Noordhuis) -2010-05-11 load objects encoded from disk directly without useless conversion (antirez) -2010-05-11 fixed a problem leading to crashes, as keys can't be currently specially encoded, so we can't encode integers at object loading time... For now this can be fixed passing a few flags, or later can be fixed allowing encoded keys as well (antirez) -2010-05-11 long long to string conversion speedup applied in other places as well. Still the code has bugs, fixing right now... (antirez) -2010-05-11 hand written code to turn a long long into a string -> very big speed win (antirez) -2010-05-11 added specialized function to compare string objects for perfect match that is optimized for this task (antirez) -2010-05-11 better use of encoding inforamtion in dictEncObjKeyCompare (antirez) -2010-05-10 CONFIG now can change appendfsync policy at run time (antirez) -2010-05-10 CONFIG command now supports hot modification of RDB saving parameters. (antirez) -2010-05-10 while loading the rdb file don't add the key to the dictionary at all if it's already expired, instead of removing it just after the insertion. (antirez) -2010-05-10 Merge branch 'check-aof' of git://github.com/pietern/redis (antirez) -2010-05-08 minor changes to improve code readability (antirez) -2010-05-08 swap objects out directly while loading an RDB file if we detect we can't stay in the vm max memory limits anyway (antirez) -2010-05-07 change command names no longer used to zunion/zinter (Pieter Noordhuis) -2010-05-07 DEBUG POPULATE command for fast creation of test databases (antirez) -2010-05-07 update TODO (Pieter Noordhuis) -2010-05-07 swap arguments in blockClientOnSwappedKeys to be consistent (Pieter Noordhuis) -2010-05-07 added function that preloads all keys needed to execute a MULTI/EXEC block (Pieter Noordhuis) -2010-05-07 add sanity check to zunionInterBlockClientOnSwappedKeys, as the number of keys used is provided as argument to the function (Pieter Noordhuis) -2010-05-07 make prototype of custom function to preload keys from the vm match the prototype of waitForMultipleSwappedKeys (Pieter Noordhuis) -2010-05-07 extract preloading of multiple keys according to the command prototype to a separate function (Pieter Noordhuis) -2010-05-07 make append only filename configurable (Pieter Noordhuis) -2010-05-07 don't load value from VM for EXISTS (Pieter Noordhuis) -2010-05-07 swap file name pid expansion removed. Not suited for mission critical software... (antirez) -2010-05-07 Swap file is now locked (antirez) -2010-05-06 Merge branch 'master' into aof-speedup (antirez) -2010-05-06 log error and quit when the AOF contains an unfinished MULTI (antirez) -2010-05-06 log error and quit when the AOF contains an unfinished MULTI (Pieter Noordhuis) -2010-05-06 Merge branch 'master' into check-aof (Pieter Noordhuis) -2010-05-06 hincrby should report an error when called against a hash key that doesn't contain an integer (Pieter Noordhuis) -2010-05-06 AOF writes are now accumulated into a buffer and flushed into disk just before re-entering the event loop. A lot less writes but still this guarantees that AOF is written before the client gets a positive reply about a write operation, as no reply is trasnmitted before re-entering into the event loop. (antirez) -2010-05-06 clarified a few messages in redis.conf (antirez) -2010-05-05 ask for confirmation before AOF is truncated (Pieter Noordhuis) -2010-05-05 str can be free'd outside readString (Pieter Noordhuis) -2010-05-05 moved argument parsing around (Pieter Noordhuis) -2010-05-05 ignore redis-check-aof binary (Pieter Noordhuis) -2010-05-05 allow AOF to be fixed by truncating to the portion of the file that is valid (Pieter Noordhuis) -2010-05-05 tool to check if AOF is valid (Pieter Noordhuis) -2010-05-02 included fmacros.h in linenose.c to avoid compilation warnings on Linux (antirez) -2010-05-02 compilation fix for mac os x (antirez) -2010-05-02 Merge branch 'master' of git@github.com:antirez/redis (antirez) -2010-05-02 On Linux now fdatasync() is used insetad of fsync() in order to flush the AOF file kernel buffers (antirez) -2010-04-30 More tests for APPEND and tests for SUBSTR (antirez) -2010-04-30 linenoise.c updated, now redis-cli can be used in a pipe (antirez) -2010-04-29 redis-cli minor fix (less segfault is better) (antirez) -2010-04-29 New MONITOR output format with timestamp, every command in a single line, string representations (antirez) -2010-04-29 redis-cli INFO output format is now raw again (antirez) -2010-04-29 Added more information about slave election in Redis Cluster alternative doc (antirez) -2010-04-29 Redis cluster version 2 (antirez) -2010-04-27 Fixed a redis-cli bug, was using free instead of zfree call (antirez) -2010-04-27 AOF is now rewritten on slave after SYNC with master. Thanks to @_km for finding this bug and any others' (antirez) -2010-04-27 redis-cli is now using only the new protocol (antirez) -2010-04-27 Minimal support for subscribe/psubscribe in redis-cli (antirez) -2010-04-26 don't output the newline when stdout is not a tty (antirez) -2010-04-26 redis-cli now is able to also output the string representation instead of the raw string. Much better for debugging (antirez) -2010-04-26 Initial support for quoted strings in redis-cli (antirez) -2010-04-23 SETEX implemented (antirez) -2010-04-23 Pub/Sub API change: now messages received via pattern matching have a different message type and an additional field representing the original pattern the message matched (antirez) -2010-04-22 typo fixed, reloaded (antirez) -2010-04-22 typo fixed (antirez) -2010-04-22 REDIS-CLUSTER doc updated (antirez) -2010-04-22 Virtual memory design document removed, no longer needed as we have a full specification and implementation (antirez) -2010-04-22 new units for bytes specification (antirez) -2010-04-22 Now in redis.conf it is possible to specify units where appropriate instead of amounts of bytes, like 2Gi or 4M and so forth (antirez) -2010-04-21 binary safe keys ready implementation of RANDOMKEYS (antirez) -2010-04-21 Now that's the right 1.3.10 (antirez) -2010-04-21 Revert "fsync always now uses O_DIRECT on Linux" (antirez) -2010-04-21 Revert "define __USE_GNU to get O_DIRECT" (antirez) -2010-04-21 Merge branch 'master' of github.com:antirez/redis (antirez) -2010-04-21 Revert "version 1.3.10" (antirez) -2010-04-21 version 1.3.10 (antirez) -2010-04-20 define __USE_GNU to get O_DIRECT (antirez) -2010-04-20 fsync always now uses O_DIRECT on Linux (antirez) -2010-04-20 More precise memory used guesswork in zmalloc.c (antirez) -2010-04-19 Fix for MULTI/EXEC and Replication/AOF: now the block is correctly sent as MULTI/..writing operations../EXEC. Ok for slaves but more work needed for the AOF as it should be a write-all-or-nothing business (antirez) -2010-04-19 running the test using tcl8.5 directly instead of tclsh that too often it's a symlink to 8.4 (antirez) -2010-04-19 Added package require Tcl 8.5 in redis.tcl so it will show a clear error when the test suit is attempted to run under 8.4 (antirez) -2010-04-18 Fix for a SORT bug introduced with commit 16fa22f1, regression test added (antirez) -2010-04-18 Guru mediation -> meditation (antirez) -2010-04-16 check eptr inline (Pieter Noordhuis) -2010-04-16 refactor code that retrieves value from object or replies to client (Pieter Noordhuis) -2010-04-17 Merge branch 'hash' of git://github.com/pietern/redis (antirez) -2010-04-17 redisAssert(0) => redisPanic("something meaningful") (antirez) -2010-04-17 make sure that the resulting value in hincrby is encoded when possible (Pieter Noordhuis) -2010-04-17 increment dirty counter after hmset (Pieter Noordhuis) -2010-04-17 strip tryObjectEncoding from hashSet, to enable the arguments being encoded in-place (Pieter Noordhuis) -2010-04-17 Added support for Guru Mediation, and raising a guru mediation if refCount <= 0 but decrRefCount is called against such an object (antirez) -2010-04-16 fix small error and memory leaks in SORT (Pieter Noordhuis) -2010-04-16 SORT/GET test added (antirez) -2010-04-16 Added tests for GET/BY against hashes fields (antirez) -2010-04-16 Merge branch 'hash-refactor' of git://github.com/pietern/redis (antirez) -2010-04-16 check object type in lookupKeyByPattern (Pieter Noordhuis) -2010-04-16 make sortCommand aware that lookupKeyByPattern always increased the refcount of the returned value (Pieter Noordhuis) -2010-04-16 revert 0c390a to stop using tricks with o->refcount (Pieter Noordhuis) -2010-04-16 store the hash iterator on the heap instead of the stack (Pieter Noordhuis) -2010-04-16 drop inline directive (Pieter Noordhuis) -2010-04-16 rename hashReplace to hashSet (Pieter Noordhuis) -2010-04-16 added dictFetchValue() to dict.c to make hash table API a bit less verbose in the common cases (antirez) -2010-04-03 Don't set expire to keys with ttl=0, remove them immediately. (antirez) -2010-04-15 make sure that cmpobj is in decoded form when sorting by ALPHA (this solves edge case from previous commit where (!sortby && alpha) == 1) (Pieter Noordhuis) -2010-04-15 enable hash dereference in SORT on BY and GET (Pieter Noordhuis) -2010-04-15 use shared replies for hset (Pieter Noordhuis) -2010-04-15 set refcount of string objects retrieved from zipmaps to 0, so we don't have to touch the refcount of the objects inside dicts (Pieter Noordhuis) -2010-04-15 added HSETNX (Pieter Noordhuis) -2010-04-14 refactor of hash commands to use specialized api that abstracts zipmap and dict apis (Pieter Noordhuis) -2010-04-13 move retrieval of long up to prevent an empty hash from being created (Pieter Noordhuis) -2010-04-15 more advanced leaks detection in test redis (antirez) -2010-04-15 ability to select port/host from make test (antirez) -2010-04-15 Active rehashing (antirez) -2010-04-15 Incrementally rehahsing hash table! Thanks to Derek Collison and Pieter Noordhuis for feedbacks/help (antirez) -2010-04-14 Does not allow commands other than Pub/Sub commands when there is at least one pattern (antirez) -2010-04-13 Fixed a tiny memory leak when loading the configuration file. (Alex McHale) -2010-04-13 Merge branch 'hmget' of git://github.com/pietern/redis (antirez) -2010-03-29 Validate numeric inputs. (Alex McHale) -2010-03-24 Remove trailing whitespace. (Alex McHale) -2010-04-12 Now all the commands returning a multi bulk reply against non existing keys will return an empty multi bulk, not a nil one (antirez) -2010-04-12 implemented HMGET (Pieter Noordhuis) -2010-04-12 implemented HMSET (Pieter Noordhuis) -2010-04-12 Sharing of small integer objects: may save a lot of memory with datasets having many of this (antirez) -2010-04-10 dict.c fixed to play well with enabling/disabling of the hash table (antirez) -2010-04-09 removed a no longer true assert in the VM code (antirez) -2010-04-09 shareobjects feautres killed - no gains most of the time, but VM complexities (antirez) -2010-04-09 use directly the real key object in VM I/O jobs to match by pointer, and to handle different keys with the same name living in different DBs, but being at the same moment in the IO job queues (antirez) -2010-04-08 last change reverted as it was unstable... more testing needed (antirez) -2010-04-08 Prevent hash table resize while there are active child processes in order to play well with copy on write (antirez) -2010-04-08 Merge branch 'issue_218' of git://github.com/pietern/redis (antirez) -2010-04-08 -1 not needed... (antirez) -2010-04-08 Skiplist theoretical fix (antirez) -2010-04-07 Now when a child is terminated by a signal, the signal number is logged as well (antirez) -2010-04-07 First version of evented Redis Tcl client, that will be used for BLPOP and Pub/Sub tests (antirez) -2010-04-05 use long long reply type for HINCRBY (Pieter Noordhuis) -2010-04-05 last argument is never encoded for HINCRBY (Pieter Noordhuis) -2010-04-02 Now PUBLISH commands are replicated to slaves (antirez) -2010-04-01 use the right object when cleaning up after zunion/zinter (fixes issue 216) (Pieter Noordhuis) -2010-04-01 Merge branch 'zipmap' of git://github.com/pietern/redis (antirez) -2010-04-01 reduce code complexity because zipmapLen now is O(1) (Pieter Noordhuis) -2010-04-01 update the zipmap entry in-place instead of appending it (Pieter Noordhuis) -2010-04-01 updated zipmap documentation to match the implementation (Pieter Noordhuis) -2010-04-01 allow 4 free trailing bytes for each value (Pieter Noordhuis) -2010-04-01 Pub/Sub pattern matching capabilities (antirez) -2010-04-01 use function to determine length of a single entry (Pieter Noordhuis) -2010-03-31 Deny EXEC under out of memory (antirez) -2010-03-29 No timeouts nor other commands for clients in a Pub/Sub context (antirez) -2010-03-29 free hash table entries about no longer active classes, so that PUBSUB can be abused with millions of different classes (antirez) -2010-03-29 Fixed a refcount stuff leading to PUBSUB crashes (antirez) -2010-03-29 fmacros added to linenoise, avoiding all the nice warnings... (antirez) -2010-03-29 First pubsub fix (antirez) -2010-03-29 PUBSUB implemented (antirez) -2010-03-29 Redis version is now 1.3.8 (antirez) -2010-03-28 removed references in code to ZIPMAP_EMPTY (Pieter Noordhuis) -2010-03-28 use first byte of zipmap to store length (Pieter Noordhuis) -2010-03-28 implemented strategy that doesn't use free blocks in zipmaps (Pieter Noordhuis) -2010-03-26 Merge branch 'hincrby' of git://github.com/pietern/redis (antirez) -2010-03-26 removed unnecessary refcount increase that caused the HINCRBY memleak (Pieter Noordhuis) -2010-03-26 implements HINCRBY and tests (todo: find and fix small memleak) (Pieter Noordhuis) -2010-03-26 Removed a useless if spotted by Pieter Noordhuis (antirez) -2010-03-26 Fixed a critical replication bug: binary values issued with the multi bulk protocol caused a protocol desync with slaves. (antirez) -2010-03-24 Fixed the reply about denied write commands under maxmemory reached condition: now the error will no longer lead to a client-server protocol desync (antirez) -2010-03-24 CONFIG command implemened -- just a start but already useful (antirez) -2010-03-24 redis-cli prompt is now redis> (antirez) -2010-03-23 with --help states that you can use - as config file name to feed config via stdin (antirez) -2010-03-23 New INFO field: expired_keys (antirez) -2010-03-23 the Cron timer function is now called 10 times per second instead of 1 time per second to make Redis more responsibe to BGSAVE and to delete expired keys more incrementally (antirez) -2010-03-23 Use linenoise for line editing on redis-cli. (Michel Martens) -2010-03-23 Fix authentication for redis-cli on non-interactive mode. (Michel Martens) -2010-03-23 key deletion on empty value fix + some refactoring (antirez) -2010-03-23 Empty value trigger key removal in all the operations (antirez) -2010-03-22 Merged gnrfan patches fixing issues 191, 193, 194 (antirez) -2010-03-22 Merge branch 'issue_193' of git://github.com/gnrfan/redis (antirez) -2010-03-22 Merge branch 'issue_191' of git://github.com/gnrfan/redis (antirez) -2010-03-22 Redis master version is now 1.3.7 (antirez) -2010-03-19 support for include directive in config parser (Jeremy Zawodny) -2010-03-19 Removed a stupid overriding of config values due to a wrong cut&paste (antirez) -2010-03-19 VM hash type swappability implemented. Handling of failed pthread_create() call. (antirez) -2010-03-19 Solving issue #191 on Google Code: -v and --version should print the version of Redis (Antonio Ognio) -2010-03-19 Solves issue #194 on Google Code: --help parameter to redis-srver prints the usage message (Antonio Ognio) -2010-03-19 Fixing issue 193 (Antonio Ognio) -2010-03-18 increment server.dirty on HDEL (antirez) -2010-03-18 Redis 1.3.6 (antirez) -2010-03-18 test-redis.tcl dataset digest function Hash support (antirez) -2010-03-18 zipmap fix for large values (antirez) -2010-03-18 Optimization fixed and re-activated (antirez) -2010-03-18 reverted an optimization that makes Redis not stable (antirez) -2010-03-18 Fixed redis-cli auth code (antirez) -2010-03-17 HDEL fix, an optimization for comparison of objects in hash table lookups when they are integer encoding (antirez) -2010-03-17 Version is now 1.3.5 (antirez) -2010-03-17 Merged Pietern patch for VM key args helper function. Fixed an obvious bug in the redis-cli passwd auth stuff (antirez) -2010-03-17 Merge branch 'aggregates' of git://github.com/pietern/redis (antirez) -2010-03-17 Added Authentication to redis-cli.c using -a switch Update usage fixed Makefile to delete redis-check-dump during make clean (root) -2010-03-17 HEXISTS and tests implemented (antirez) -2010-03-17 More hash tests (antirez) -2010-03-17 better HSET test (antirez) -2010-03-17 Fixed a bug in HSET, a memory leak, and a theoretical bug in dict.c (antirez) -2010-03-17 More Hash tests (antirez) -2010-03-13 added preloading keys from VM when using ZINTER or ZUNION (Pieter Noordhuis) -2010-03-13 added explicit AGGREGATE [SUM|MIN|MAX] option to ZUNION/ZINTER (Pieter Noordhuis) -2010-03-16 HGET fix for integer encoded field against zipmap encoded hash (antirez) -2010-03-16 zrevrank support in redis-cli (antirez) -2010-03-16 HKEYS / HVALS / HGETALL (antirez) -2010-03-16 Solved a memory leak with Hashes (antirez) -2010-03-15 pretty big refactoring (antirez) -2010-03-15 An interesting refactoring + more expressive internal API (antirez) -2010-03-15 Fixed the same problem in ZREVRANK (antirez) -2010-03-15 Fixed a ZRANK bug (antirez) -2010-03-15 zipmap to hash conversion in HSET (antirez) -2010-03-14 max zipmap entries and max zipmap value parameters added into INFO output (antirez) -2010-03-14 HDEL and some improvement in DEBUG OBJECT command (antirez) -2010-03-14 Append only file support for hashes (antirez) -2010-03-13 utility to check rdb files for unprocessable opcodes (Pieter Noordhuis) -2010-03-12 A minor fix and a few debug messages removed (antirez) -2010-03-12 Applied the replication bug patch provided by Jeremy Zawodny, removing temp file collision after the slave got the dump.rdb file in the SYNC stage (antirez) -2010-03-11 Fix for HGET against non Hash type, debug messages used to understand a bit better a corrupted rdb file (antirez) -2010-03-09 fix: use zmalloc instead of malloc (Pieter Noordhuis) -2010-03-09 Merged zsetops branch from Pietern (antirez) -2010-03-09 Merged ZREMBYRANK from Pietern (antirez) -2010-03-09 Merged ZREVRANK from Pietern (antirez) -2010-03-09 use a struct to store both a dict and its weight for ZUNION and ZINTER, so qsort can be applied (Pieter Noordhuis) -2010-03-09 Hash auto conversion from zipmap to hash table, type fixed for hashes, hash loading from disk (antirez) -2010-03-09 replaced ZMERGE by ZUNION and ZINTER. note: key preloading by the VM does not yet work (Pieter Noordhuis) -2010-03-08 Hashes saving / fixes (antirez) -2010-03-08 use ZMERGE as starting point (Pieter Noordhuis) -2010-03-07 HSET fixes, now the new pointer is stored back in the object pointer field (antirez) -2010-03-07 added ZREVRANK (Pieter Noordhuis) -2010-03-06 Fix for replicaiton with over 2GB dump file initial SYNC stage (antirez) -2010-03-06 first implementation of HSET/HSET. More work needed (antirez) -2010-03-05 zipmaps functions to get, iterate, test for existence. Initial works for Hash data type (antirez) -2010-03-04 redis-benchmark now implements Set commands benchmarks (antirez) -2010-03-04 zipmap iteration code (antirez) -2010-03-04 moved code to delete a single node from a zset to a separate function (Pieter Noordhuis) -2010-03-04 rename zslDeleteRange to zslDeleteRangeByScore (to differentiate between deleting using score or rank) (Pieter Noordhuis) -2010-03-04 use 1-based rank across zsl*Rank functions consistently (Pieter Noordhuis) -2010-03-04 implemented ZREMBYRANK (Pieter Noordhuis) -2010-03-04 A fix for initialization of augmented skip lists (antirez) -2010-03-04 A fix for an invalid access when VM is disabled (antirez) -2010-03-04 Merge branch 'zsl-get-rank' of git://github.com/pietern/redis (antirez) -2010-03-04 redis-cli now runs in interactive mode if no command is provided (antirez) -2010-03-04 merged memory reduction patch (Pieter Noordhuis) -2010-03-04 Now list push commands return the length of the new list, thanks to Gustavo Picon (antirez) -2010-03-04 first check if starting point is trivial (head or tail) before applying log(N) search (Pieter Noordhuis) -2010-03-04 use rank to find starting point for ZRANGE and ZREVRANGE (Pieter Noordhuis) -2010-03-04 lookup rank of a zset entry in a different function (Pieter Noordhuis) -2010-03-04 SUBSTR fix for integer encoded vals (antirez) -2010-03-04 fix ZRANK (realize that rank is 1-based due to the skip list header) (Pieter Noordhuis) -2010-03-03 initial implementation of SUBSTR (antirez) -2010-03-03 TODO updated (antirez) -2010-03-03 fpurge call removed from redis-cli (antirez) -2010-03-03 ZRANK stress tester (antirez) -2010-03-03 use less memory as element->span[0] will always be 1; any level 0 skip list is essentially a linked list (Pieter Noordhuis) -2010-03-03 rank is very unlikely to overflow integer range (Pieter Noordhuis) -2010-03-03 x->backward never equals zsl->header (Pieter Noordhuis) -2010-03-03 initial implementation for augmented zsets and the zrank command (Pieter Noordhuis) -2010-03-03 zipampDel() implemented (antirez) -2010-03-03 added quit and exit commands to redis-cli in order to quit the interactive mode (antirez) -2010-03-03 Merge remote branch 'djanowski/interactive' (antirez) -2010-03-02 Add support for MULTI/EXEC. (Damian Janowski & Michel Martens) -2010-03-02 Remove trailing newline in interactive mode. (Damian Janowski & Michel Martens) -2010-03-02 minor fix for a Linux warning (antirez) -2010-03-02 Add interactive mode to redis-cli. (Michel Martens & Damian Janowski) -2010-03-02 Better to increment the version minor number when a VM bug is fixed... it will be simpler to understand what's going on when users will report problems with the INFO trace. (antirez) -2010-03-02 Fixed a subtle VM bug... was not flushing the buffer so the child process read truncated data (antirez) -2010-03-01 KEYS now returns a multi bulk reply (antirez) -2010-02-27 Add DISCARD command to discard queued MULTI commands. (antirez) -2010-03-01 Swappability bug due to a typo fixed thanks to code review by Felix Geisendörfer @felixge (antirez) -2010-02-28 minor fixes for zipmap.c (antirez) -2010-02-27 first zipmap fix of a long sequence in the days to come ;) (antirez) -2010-02-27 initial zipmap.c implementation (antirez) -2010-02-27 Bug #169 fixed (BLOP/BRPOP interrupted connections are not cleared from the queue) (antirez) -2010-02-22 Fixed 32bit make target to work on Linux out of the box (antirez) -2010-02-19 A problem with replication with multiple slaves connectiong to a single master fixed. It was due to a typo, and reported on github by the user micmac. Also the copyright year fixed from many files. (antirez) -2010-02-10 Saner VM defaults for redis.conf (antirez) -2010-02-09 VM now is able to block clients on swapped keys for all the commands (antirez) -2010-02-07 ZCOUNT and ZRANGEBYSCORE new tests (antirez) -2010-02-07 ZRANGEBYSCORE now supports open intervals, prefixing double values with a open paren. Added ZCOUNT that can count the elements inside an interval of scores, this supports open intervals too (antirez) -2010-02-07 WITHSCORES in ZRANGEBYSCORE thanks to Sam Hendley (antirez) -2010-02-06 Added "withscores" option to zrangebyscore command. Based on withscores support in zrange function, ugliest part was the argument parsing to handle using it with the limit option. (Sam Hendley) -2010-02-06 DEBUG OBJECT provide info about serialized object length even when VM is disabled (antirez) -2010-02-06 multi bulk requests in redis-benchmark, default fsync policy changed to everysec, added a prefix character for DEBUG logs (antirez) -2010-02-04 APPEND tests (antirez) -2010-02-04 APPEND command (antirez) -2010-02-02 Faster version of the function hashing possibly encoded objects, leading to a general speed gain when working with Sets of integers (antirez) -2010-02-02 faster Set loading time from .rdb file resizing the hash table to the right size before loading elements (antirez) -2010-02-02 Log time taken to load the DB at startup, in seconds (antirez) -2010-01-31 Fixed VM corruption due to child fclosing the VM file directly or indirectly calling exit(), now replaced with _exit() in all the sensible places. Masked a few signals from IO threads. (antirez) -2010-01-28 loading side of the threaded VM (antirez) -2010-01-26 TODO cahnges (antirez) -2010-01-23 Fixed memory human style memory reporting, removed server.usedmemory, now zmalloc_used_memory() is used always. (antirez) -2010-01-22 VM tuning thanks to redis-stat vmstat. Now it performs much better under high load (antirez) -2010-01-21 Changelog updated (antirez) -2010-01-21 REDIS_MAX_COMPLETED_JOBS_PROCESSED is now in percentage, not number of jobs. Moved a debugging message a few lines forward as it was called where a few logged parameters where invalid, leading to a crash (antirez) -2010-01-20 fixed a deadlock caused by too much finished processes in queue so that I/O clients writing to the wirte side of the pipe used to awake the main thread where blocking. Then a BGSAVE started waiting for the last active thread to finish, condition impossible because all the I/O threads where blocking on threads. Takes this as a note to myself... (antirez) -2010-01-20 ae.c event loop does no longer support exception notifications, as they are fully pointless. Also a theoretical bug that never happens in practice fixed. (antirez) -2010-01-19 commercial tools stuff removed from the Redis makefile. cotools are now migrated into a different repos (antirez) -2010-01-19 removed a bug in the function to cancel an I/O job (antirez) -2010-01-17 static symbols update (antirez) -2010-01-16 removed support for REDIS_HELGRIND_FRIENDLY since Helgrind 3.5.0 is friendly enough even with many threads created and destroyed (antirez) -2010-01-15 now redis-cli understands -h (antirez) -2010-01-15 Create swap file only if not exists (antirez) -2010-01-15 I hate warnings (antirez) -2010-01-15 fixed a minor memory leak in configuration file parsing (antirez) -2010-01-15 minor fix (antirez) -2010-01-15 support for named VM swap file. Fixed a few important interaction issues between the background saving processes and IO threads (antirez) -2010-01-15 fix for the just added new test (antirez) -2010-01-15 useless debugging messages removed (antirez) -2010-01-15 new test added (antirez) -2010-01-15 thread safe zmalloc used memory counter (antirez) -2010-01-15 A define to make Redis more helgrind friendly (antirez) -2010-01-15 removed a few races from threaded VM (antirez) -2010-01-14 Fixed a never experienced, theoretical bug that can actually happen in practice. Basically when a thread is working on a I/O Job we need to wait it to finish before to cancel the Job in vmCancelThreadedIOJob(), otherwise the thread may mess with an object that is being manipulated by the main thread as well. (antirez) -2010-01-14 Set the new threads stack size to a LZF friendly amount (antirez) -2010-01-13 access to already freed job structure fixed by statements reoredering (antirez) -2010-01-13 removed a useless debugging message (antirez) -2010-01-13 Wait zero active threads condition before to fork() for BGSAVE or BGREWRITEAOF (antirez) -2010-01-13 list API is now thread safe (antirez) -2010-01-13 minor TODO and debugging info changes (antirez) -2010-01-12 support for blocking VM in config file (antirez) -2010-01-12 more non blocking VM changes (antirez) -2010-01-12 fix for test #11 (antirez) -2010-01-12 a few more stuff in INFO about VM. Test #11 changed a bit in order to be less lame (antirez) -2010-01-12 Added a define to configure how many completed IO jobs the handler should process at every call. (antirez) -2010-01-11 Fixed a bug in the IO Job canceling funtion (antirez) -2010-01-11 more steps towards a working non blocking VM (antirez) -2010-01-11 converted random printfs in debug logs (antirez) -2010-01-11 removed a bug introduced with non blocking VM (antirez) -2010-01-11 a few non blocking VM bugs fixed (antirez) -2010-01-11 More work on non-blocking VM. Should work in a few days (antirez) -2010-01-11 More threaded I/O VM work + Redis init script (antirez) -2010-01-10 more work on VM threaded I/O. Still nothing of usable (antirez) -2010-01-09 non-blocking VM data structures, just a start (antirez) -2010-01-08 used_memory_human added to INFO output. Human readable amount of memory used. (antirez) -2010-01-07 Now DEBUG OBJECT plays well with swapped out objects (antirez) -2010-01-07 fflush VM swap file after object swapping (antirez) -2010-01-07 added the fmacros to enable support for fseeko() lseeko() with 64bit off_t (antirez) -2010-01-07 VM now swaps objects out while loading datasets not fitting into vm-max-memory bytes of RAM (antirez) -2010-01-07 added process id information in INFO (antirez) -2010-01-06 vm-enabled set to no by default in redis.conf (antirez) -2010-01-06 a new default redis.conf (antirez) -2010-01-06 VM stats in INFO command (antirez) -2010-01-06 Introduced a new log verbosity level, so now DEBUG is really for debugging. Refactored a bit maxmemory. When virtual memory is short in RAM free the objects freelist as well as swapping things out. (antirez) -2010-01-05 fixed a bug in bgsave when VM is off but still it was testing for obj->storage field (antirez) -2010-01-05 converted a few calls to assert() => redisAssert() to print stack trace (antirez) -2010-01-05 BGREWRITEAOF now works with swapping on (antirez) -2010-01-05 A first fix for SET key overwrite (antirez) -2010-01-05 SAVE now works with VM (antirez) -2010-01-05 swapping algorithm a bit more aggressive under low memory (antirez) -2010-01-05 basic VM mostly working! (antirez) -2010-01-05 New object field (one of the unused bytes) to hold the type of the swapped out value object in key objects (antirez) -2010-01-05 VM internals bugfixes, set 1 (antirez) -2010-01-05 load key from swap on key lookup (antirez) -2010-01-05 more object-level VM primitives (antirez) -2010-01-05 Redis objects swapping / loading (antirez) -2010-01-05 rdbLoadObject() as a separated function to load objects from disk. Dropped support for RDB version 0, I guess no longer has this legacy DBs around (antirez) -2010-01-04 VM low level pages handling (antirez) -2010-01-04 vm swap file creation, and some basic configuration (antirez) -2010-01-04 version marked 1.3.2 (antirez) -2010-01-04 saving code refactored a bit, added a function returning the number of bytes an object will use on disk (antirez) -2010-01-02 Now the PUSH side of RPOPLPUSH is able to unblock clients blocked on BLPOP (antirez) -2010-01-02 Version is now 1.3.1 (antirez) -2010-01-02 New vararg BLPOP able to block against multiple keys (antirez) -2009-12-29 fixed a problem with BLPOP timeout of zero, now it blocks forever (antirez) -2009-12-29 BLPOP timeouts implemented (antirez) -2009-12-29 first working implementation of BLPOP and BRPOP, still everything is to test well (antirez) -2009-12-29 a few more fixes, still broken (antirez) -2009-12-29 First fix, still broken (antirez) -2009-12-29 minor fix for Linux 64 bit (antirez) -2009-12-29 not yet working BLPOP implementation (antirez) -2009-12-27 AOFSYNC removed, got a better idea... (antirez) -2009-12-27 AOFSYNC command implemented (antirez) -2009-12-27 Version changed to 1.3.0, welcome to the new unstable (antirez) -2009-12-27 Now MULTI returns +OK as well (antirez) -2009-12-27 MULTI/EXEC first implementation (antirez) -2009-12-24 Fixed a minor bug in GETSET, now the SET part is not performed if the GET fails because the key does not contain a string value (antirez) -2009-12-23 html doc readded (antirez) -2009-12-23 ZRANGE WITHSCORES test added (antirez) -2009-12-23 version is now 1.1.94 (antirez) -2009-12-23 Add the command name in the unknown command error message. (antirez) -2009-12-22 ZRANGE, ZREVRANGE now support WITHSCORES options (antirez) -2009-12-22 html docs update (ZINCRBY added) (antirez) -2009-12-18 TODO list update (antirez) -2009-12-18 the pipelining test was ran against DB 1 for error, now it runs on DB 9 like all the other tests (antirez) -2009-12-18 still more tests (antirez) -2009-12-18 SORT STORE test added (antirez) -2009-12-18 Now SORT returns an empty bulk reply if the key does not exist (antirez) -2009-12-18 modified a bit the ZREVRANGE test to cover a few lines of code more (antirez) -2009-12-18 SHUTDOWN now does the right thing when append only is on, that is, fsync instead to save the snapshot. (antirez) -2009-12-18 Added a missing server.dirty increment in a non critical place, added more tests (antirez) -2009-12-18 LTRIM stress testing test added (antirez) -2009-12-18 LTRIM now returns +OK against non existing keys. More tests in test-redis.tcl (antirez) -2009-12-18 added sdstoupper() declaration in sds.h (antirez) -2009-12-18 Fixed sds.c bug #124 (antirez) -2009-12-16 LZF compression re-enabled by default, but with INIT_HTAB set to 0 to avoid the very costly memset initialization. Note that with this option set valgrind will output some false positive about lzf_c.c (antirez) -2009-12-16 lzf compression switched off by default now, with config file option to enable it in redis.conf (antirez) -2009-12-16 Regression for epoll bug in redis-test.tcl, version is now 1.1.93 (antirez) -2009-12-16 Fixed a lame epoll issue (antirez) -2009-12-15 html doc updated (antirez) -2009-12-15 version is now 1.1.92 (antirez) -2009-12-15 Two important fixes to append only file: zero length values and expires. A pretty neat new test to check consistency of randomly build datasets against snapshotting and AOF. (antirez) -2009-12-15 debug loadaof implemented in order to add more consistency tests in test-redis.tcl (antirez) -2009-12-15 Added a new test able to stress a lot the snapshotting engine (antirez) -2009-12-15 Unified handling of empty queries with normal queries. (antirez) -2009-12-15 Fixed some subtle bug in the command processing code almost impossible to spot in the real world, thanks to gcov (antirez) -2009-12-15 Regression test for SINTERSTORE added (antirez) -2009-12-15 Fixed issue #121 (antirez) -2009-12-14 a few more tests and ability to run a specific test in test-redis.tcl (antirez) -2009-12-13 Changed the reply of BGSAVE and BGREWRITEAOF from +OK to a more meaningful message that makes the user aware of an operation that just started and is not yet finished. (antirez) -2009-12-13 Set the master->slave logical client as authenticated on creation, so that if the slave requires a password replication works anyway (antirez) -2009-12-13 TODO update (antirez) -2009-12-12 bgrewriteaof_in_progress added to INFO (antirez) -2009-12-12 TODO list modified. What's planned for 1.4 is now written in the stone ;) (antirez) -2009-12-12 better handling of non blocking connect on redis-benchmark: EPIPE on read does not print an error message now (antirez) -2009-12-11 some change to redis-sha1.rb utility to make it more robust against non-meaningful changes in the dataset (antirez) -2009-12-10 redis-sha1.rb utility updated (antirez) -2009-12-10 a bit more verbose -ERR wrong number o arguments error, now gives info about the command name causing the error (antirez) -2009-12-10 TODO change and minor SETNX optimization (antirez) -2009-12-06 in rdbLoadDoubleValue now the buffer is nul terminated correctly. Thanks valgrind. (antirez) -2009-12-06 printf format warnings fixed by casting (antirez) -2009-12-06 Regression tests for SETNX and MSETNX bugs added (antirez) -2009-12-06 SETNX and MSETNX now respect the delete-on-write operation of EXPIREing keys (antirez) -2009-12-06 Fixed daemonization when using kqueue/kevent. Now the server initialization is performed *after* the daemonization (antirez) -2009-12-05 more HTML doc changes (antirez) -2009-12-05 HTML doc update (antirez) -2009-12-05 a few redis-cli format specified fixed (antirez) -2009-12-05 use __attribute__ format in sdscatprintf() when the compiler is GCC. Fixed format bugs resulting from the new warnings. (antirez) -2009-12-01 TODO update (antirez) -2009-12-01 compilation problem on 64bit mac os x 10.5 possibly fixed (antirez) -2009-12-01 virtual memory design doc typos (antirez) -2009-12-01 design documents added to the project (antirez) -2009-11-30 Fixed issued #85 (getDecodedObject: Assertion 1 != 1 failed. While sorting a set), added a smarter assert() function to dump the stacktrace, provided a macro to initalize Redis objects on the stack to avoid this kind of bugs. (antirez) -2009-11-30 fixed a subtle bug in redis-cli not having visible effects (antirez) -2009-11-29 TODO updated (antirez) -2009-11-29 Version chagned to 1.100, also known as the first first 2.0 beta version (antirez) -2009-11-29 more tests in test-redis.tcl, some minor fix (antirez) -2009-11-29 SORT support for sorted sets (antirez) -2009-11-28 Implemented LIMIT option in ZRANGEBYSCORE. We now enter feature-freeze (antirez) -2009-11-28 Changelog updated (antirez) -2009-11-28 html doc updated (antirez) -2009-11-28 enable kqueue/kevent only for Mac OS X 10.6.x as it seems that 10.5.x has a broken implementation of this syscalls. (antirez) -2009-11-28 TODO updated (antirez) -2009-11-28 ZRANGEBYSCORE fuzzy test (antirez) -2009-11-28 ZRANGEBYSCORE memory leak fixed, ZRANGEBYSCORE initial test added (antirez) -2009-11-28 INFO refactored. Stack trace on memory corruption now dumps the same information as the INFO command (antirez) -2009-11-28 ifdefs added to use kevent on Free Open and Net BSD as well. INFO and ae.c modified in order to report the multiplexing API in use (antirez) -2009-11-28 Enabled object encoding for multiple keys in MSET. Added a test for memory leaks in test-redis.tcl when running on Mac OS X (antirez) -2009-11-28 Merge branch 'kqueue' of git://github.com/mallipeddi/redis (antirez) -2009-11-28 Changes to TODO list, commented a function in redis.c (antirez) -2009-11-28 Added support for kqueue. (Harish Mallipeddi) -2009-11-27 TODO updated (antirez) -2009-11-26 zero length bulk data reading fixed in loadAppendOnlyFile() (antirez) -2009-11-26 append only file fixes (antirez) -2009-11-26 log rebuilding, random refactoring, work in progress please wait for an OK commit before to use this version (antirez) -2009-11-24 DEBUG RELOAD implemented, and test-redis.tcl modified to use it to check for persistence consistency. (antirez) -2009-11-24 Redis version set to 1.07 (antirez) -2009-11-24 sorted sets saving fixed (antirez) -2009-11-24 minor TODO change (antirez) -2009-11-24 minor fix to avoid a false valgrind warning. (antirez) -2009-11-23 epoll support enabled by default for Linux builds (antirez) -2009-11-23 epoll module for ae.c implemented. Some more testing needed (antirez) -2009-11-23 commented the HAVE_EPOLL test in config.h to allow compilation under Linux now that the epoll module is still missing (antirez) -2009-11-23 ae_select module added (antirez) -2009-11-23 ae.c now supports multiple polling API modules, even if only ae_select.c is implemented currently. Also adding and removing an event is now O(1). (antirez) -2009-11-23 ae.c initial refactoring for epoll implementation (antirez) -2009-11-21 version incremented up to 1.06 (antirez) -2009-11-21 TODO aesthetic changes (antirez) -2009-11-21 TODO updated with plans up to 1.5 (antirez) -2009-11-21 SRANDMEMBER test (antirez) -2009-11-21 Fixed a SORT memory leak that should never happen in practice (antirez) -2009-11-21 SORT GET # implemented, with a test (antirez) -2009-11-21 EXPIREAT test (antirez) -2009-11-20 EXPIRE tests (antirez) -2009-11-20 more RPOPLPUSH tests (antirez) -2009-11-20 RPOPLPUSH tests added (antirez) -2009-11-20 ZINCRBY return value fixed (antirez) -2009-11-20 ZINCRSCOREBY => ZINCRBY (antirez) -2009-11-19 ZINCRSCOREBY implemented (antirez) -2009-11-19 writev() finally uncommented again (antirez) -2009-11-19 redis-benchmark hopefully last bug with multi bulk reply fixed (antirez) -2009-11-19 debug mode in redis-bench (antirez) -2009-11-19 Use writev(2) if glue output buffers is disabled (antirez) -2009-11-19 benchmark.c fixes (antirez) -2009-11-18 more experiments with long replies, glue output buffer, and writev. (antirez) -2009-11-18 benchmarking with different number of LRANGE elements. Ability to change the glue output buffer limit by #define (antirez) -2009-11-18 more writev tests/work (antirez) -2009-11-18 redis-benchmark multi bulk reply support hopefully fixed (antirez) -2009-11-17 support for writev implemented but currently ifdef-ed in order to understan why I can't see the improvements expected. Btw code provided by Stefano Barbato (antirez) -2009-11-17 multi-bulk reply support for redis-bench, and as a result LRANGE is not tested, providing some number for the tuning of multi-bulk requests performances server-side (antirez) -2009-11-12 Solaris fix thanks to Alan Harder (antirez) -2009-11-12 Merge git://github.com/ianxm/redis (antirez) -2009-11-12 ZSCORE fixed, now returns NULL on missing key or missing element (antirez) -2009-11-12 Redis test will not fail the SAVE test even if a background save is in progress (antirez) -2009-11-12 LPOPPUSH renamed into RPOPLPUSH (antirez) -2009-11-11 can select db num (ian) -2009-11-11 Workaround for test-redis.tcl and Tcl 8.4.x about ZSCORE test (antirez) -2009-11-11 Removed a long time warning compiling with recent GCC on Linux (antirez) -2009-11-11 TODO updated (antirez) -2009-11-11 LPUSHPOP first implementation (antirez) -2009-11-10 Tcl script, make target, and redis.c changes to build the static symbol table automagically (antirez) -2009-11-10 Implemented a much better lazy expiring algorithm for EXPIRE (antirez) -2009-11-10 Fixed issue 92 in redis: redis-cli (nil) return value lacks CR/LF (antirez) -2009-11-10 Minor TODO change with new expiring algorithm description. New expiring algorithm moved since it'll go in 1.1 (antirez) -2009-11-04 redis-test is now a better Redis citizen, testing everything against DB 9 and 10 and only if this DBs are empty. (antirez) -2009-11-04 fixed a refcounting bug with SORT ... STORE leading to random crashes (root) -2009-11-04 masterauth option merged, thanks to Anthony Lauzon (antirez) -2009-11-03 ZSets double to string serialization fixed (antirez) -2009-11-03 client-libraries directory readded (antirez) -2009-11-03 redis.tcl put at toplevel since it's uesd for the test-redis.tcl script (antirez) -2009-11-03 client libs removed from Redis git (antirez) -2009-11-03 redis-cli now accepts a -r (repeat) switch. Still there is a memory leaks to fix (antirez) -2009-11-01 TODO updated again (antirez) -2009-11-01 TODO updated (antirez) -2009-11-01 redis-cli now makes clear when the returned string is an integer (antirez) -2009-11-01 SORT STORE option (antirez) -2009-11-01 now Redis prints DB stats just after the startup without to wait a second for the first report (antirez) -2009-11-01 another fix for append only mode, now read-only operations are not appended (antirez) -2009-11-01 appendfsync parsing in config file fixed. If you benchmarked Redis against different appendfsync options is time to try again ;) (antirez) -2009-11-01 append only file loading fixed (antirez) -2009-11-01 first version of append only file loading -- STILL BROKEN don't use it (antirez) -2009-10-31 Fixed Issue 83:Using TYPE on a zset results in a malformed response from the Redis server (antirez) -2009-10-31 Fixed compilation on Linux (antirez) -2009-10-30 append only mode is now able to translate EXPIRE into EXPIREAT transparently (antirez) -2009-10-30 appendfsync is now set to NO by default (antirez) -2009-10-30 support for appendonly mode no, always, everysec (antirez) -2009-10-30 first fix for append only mode (antirez) -2009-10-30 Initial implementation of append-only mode. Loading still not implemented. (antirez) -2009-10-30 EXPIRE behaviour changed a bit, a negative TTL or an EXPIREAT with unix time in the past will now delete the key. It seems saner to me than doing nothing. (antirez) -2009-10-30 EXPIREAT implemented, will be useful for the append-only mode (antirez) -2009-10-29 Fixed Issue 74 (ERR just returned on invalid password), now the error message is -ERR invalid password. (antirez) -2009-10-29 Fixed issue 72 (SLAVEOF shutdowns redis-server on malformed reply) (antirez) -2009-10-29 Fixed issue 77 (Incorrect time in log files) thanks to youwantalex (antirez) -2009-10-29 Fixed Issue 76 (redis-server crashes when it can't connect to MASTER and client connects to SLAVE) (antirez) -2009-10-29 ZREMRANGEBYSCORE implemented. Remove a range of elements with score between min and max (antirez) -2009-10-28 TODO changes and mostly theoretical minor skiplist change (antirez) -2009-10-28 ZLEN renamed ZCARD for consistency with SCARD (antirez) -2009-10-27 TODO reworked to reflect the real roadmap (antirez) -2009-10-27 Fix for 'make 32bit' (antirez) -2009-10-27 a fix for the solaris fix itself ;) (antirez) -2009-10-27 More Solaris fixes (antirez) -2009-10-27 A lot of ZSETs tests implemented, and a bug fixed thanks to this new tests (antirez) -2009-10-27 zmalloc Solaris fixes thanks to Alan Harder (antirez) -2009-10-27 ZSCORE implemented (antirez) -2009-10-26 fix for ZRANGEBYSCORE (antirez) -2009-10-26 ZRANGEBYSCORE implemented. Redis got range queries! (antirez) -2009-10-26 A trivial change makes the new implementation O(log(N)) instead of O(log(N))+O(M) when there are M repeated scores! (antirez) -2009-10-26 ZSET now saved on disk like any other type (antirez) -2009-10-26 double serialization routines implemented (antirez) -2009-10-26 ZSETs random fixes. Now the implementation appears to be pretty stable (antirez) -2009-10-26 another leak fixed. Can't find more for now, but still a bug in ZSETs to fix (antirez) -2009-10-26 ZSETs memory leak #1 solved, another one missing (antirez) -2009-10-26 Fix for skiplists backward link (antirez) -2009-10-26 Merged Solaris patches provided by Alan Harder (antirez) -2009-10-26 backward support to skiplists for ZREVRANGE, still broken, committing since I've to merge the Solaris patches (antirez) -2009-10-26 TODO updated (antirez) -2009-10-26 ZREM implemented (antirez) -2009-10-24 fix for ZADD in score update mode (antirez) -2009-10-24 some work on ZADD against existing element (score update), still broken... (antirez) -2009-10-23 zrange now starts to work. zadd still does not support update and will crash or leak or b000mmmmm (antirez) -2009-10-23 zrange initial hack (not working for now) (antirez) -2009-10-23 first skiplist fix, courtesy of valgrind (antirez) -2009-10-23 zset symbols added to stack trace code. ZSets will simply crash at the moment (antirez) -2009-10-23 more work on ZSETs and a new make target called 32bit to build i386 binaries on mac os x leopard (antirez) -2009-10-23 initial skiplist implementation. Most memory checks removed and zmalloc() modified to fail with an error message and abort. Anyway Redis is not designed to recover from out of memory conditions. (antirez) -2009-10-23 Fixed compilation in mac os x snow leopard when compiling a 32 bit binary. (antirez) -2009-10-22 version incremented to 1.050 to distinguish from 1.001 stable and next stable versions with minor fixes (antirez) -2009-10-21 TODO updated (antirez) -2009-10-21 SRANDMEMBER added (antirez) -2009-10-20 Imporant bug leading to data corruption fixed (NOT affecting stable distribution), Tcl client lib MSET/MSETNX implementation fixed, Added new tests for MSET and MSETNX in test-redis.tcl (antirez) -2009-10-17 added multi-bulk protocol support to redis-cli and support for MSET and MSETNX (antirez) -2009-10-17 MSET fixed, was not able to replace keys already set for a stupid bug (antirez) -2009-10-16 some dead code removed (antirez) -2009-10-16 multi bulk input protocol fixed (antirez) -2009-10-16 MSET and MSETNX commands implemented (antirez) -2009-10-07 undoed all the sds hacking that lead just to random bugs and no memory saving ;) (antirez) -2009-10-07 initial multi-bulk query protocol, this will allow MSET and other interesting features. (antirez) -2009-10-03 benchmark now outputs the right command line to shorten the TIME_WAIT interval on Mac OS X when keep alive is set (antirez) -2009-10-02 Issue 69 fixed. Object integer encoding now works with replication and MONITORing again. (antirez) -2009-09-18 LREM fixed, used to crash since the new object integer encoding is on the stage (antirez) -2009-09-17 maxmemory didn't worked in 64 systems for values > 4GB since it used to be an unsigned int. Fixed (antirez) -2009-09-10 incremented version number to 1.001, AKA Redis edge is no longer stable... (antirez) -2009-09-10 in-memory specialized object encoding (for now 32 signed integers only) (antirez) -2009-09-03 Latest doc changes for 1.0 (antirez) -2009-09-03 Redis 1.0.0 release (antirez) -2009-09-02 Redis version pushed to 1.0 (antirez) -2009-09-02 Ruby client lib updated to the latest git version (antirez) -2009-09-02 update-scala-client script added (antirez) -2009-09-02 Scala client added thanks to Alejanro Crosa (antirez) -2009-09-02 QuickStart added (antirez) -2009-09-01 Fixed crash with only space and newline as command (issue 61), thanks to a guy having as nick "fixxxerrr" (antirez) -2009-08-11 TODO list modified (antirez) -2009-07-24 more snow leopard related fixes (for 32bit systems) (antirez) -2009-07-24 fixed compilation with Snow Leopard, thanks to Lon Baker for providing SSH access to Snow Leopard box (antirez) -2009-07-22 Fixed NetBSD compile problems (antirez) -2009-07-17 now the size of the shared pool can be really modified via config, also the number of objects in the sharing pool is logged when the log level is set to debug. Thanks to Aman Gupta (antirez) -2009-07-05 added utils/redis-copy.rb, a script that is able to copy data from one Redis server to another one on the fly. (antirez) -2009-07-04 Applied three different patches thanks to Chris Lamb, one to fix compilation and get the IP register value on Linux IA64 and other systems. One in order to log the overcommit problem on the logs instead of the standard output when Redis is demonized. The latest in order to suggest a more consistent way in order to switch to 1 the memory overcommit Linux feature. (antirez) -2009-07-03 bugfix: EXPIRE now propagates to the Slave. (antirez) -2009-06-16 Redis version modified to 0.900 (antirez) -2009-06-16 update-ruby-client script already points to ezmobius repo (antirez) -2009-06-16 client libraries updated (antirez) -2009-06-16 Redis release candidate 1 (antirez) -2009-06-16 Better handling of background saving process killed or crashed (antirez) -2009-06-14 number of keys info in INFO command thanks to Diego Rosario Brogna (antirez) -2009-06-14 SPOP documented (antirez) -2009-06-14 Clojure library thanks to Ragnar Dahlén (antirez) -2009-06-10 It is now possible to specify - as config file name to read it from stdin (antirez) -2009-06-10 sync with jodosha redis-rb (antirez) -2009-06-10 Redis-rb sync (antirez) -2009-06-10 max inline request raised again to 1024*1024*256 bytes (antirez) -2009-06-10 max bytes in an inline command raised to 1024*1024 bytes, in order to allow for very large MGETs and still protect from client crashes (antirez) -2009-06-08 SPOP implemented. Hash table resizing for Sets and Expires too. Changed the resize policy to play better with RANDOMKEY and SPOP. (antirez) -2009-06-07 some minor changes to the backtrace code (antirez) -2009-06-07 enable backtrace capabilities only for Linux and MacOSX (antirez) -2009-06-07 Dump a backtrace on sigsegv/sigbus, original coded thanks to Diego Rosario Brogna, modified in order to work on different OSes and to enhance reliability (antirez) -2009-06-06 Merge git://github.com/dierbro/redis (antirez) -2009-06-06 add more output (hrothgar) -2009-06-06 store static function pointer for a useful stack trace (hrothgar) -2009-06-06 TODO updated (antirez) -2009-06-06 Makefile dependencies updated (antirez) -2009-06-05 Avoid a busy loop while sending very large replies against very fast links, this allows to be more responsive with other clients even under a KEY * against the loopback interface (antirez) -2009-06-05 Kill the background saving process before performing SHUTDOWN to avoid races (antirez) -2009-06-05 LREM now returns :0 for non existing keys (antirez) -2009-06-05 - put some order in code - better output (hrothgar) -2009-06-05 added config.h for #ifdef business isolation, added fstat64 for Mac OS X (antirez) -2009-06-04 remove die() :-) (hrothgar) -2009-06-04 add compile options to debug (hrothgar) -2009-06-04 initial commit print stack trace (hrothgar) -2009-06-04 initial commit print stack trace (hrothgar) -2009-06-04 macosx specific zmalloc.c, uses malloc_size function in order to avoid to waste memory and time to put an additional header (antirez) -2009-06-04 DEBUG OBJECT implemented (antirez) -2009-06-04 backtrace support removed: unreliable stack trace :( (antirez) -2009-06-04 initial backtrace dumping on sigsegv/sigbus + debug command (antirez) -2009-06-03 Python lib updated (antirez) -2009-06-03 shareobjectspoolsize implemented in reds.conf, in order to control the pool size when object sharing is on (antirez) -2009-05-30 Erlang client updated (antirez) -2009-05-30 Python client library updated (antirez) -2009-05-29 Redis-rb minor bool convertion fix (antirez) -2009-05-29 ruby library client is not Redis-rb merged with RubyRedis "engine" by Brian McKinney (antirez) -2009-05-28 __P completely removed from pqsort.c/h (antirez) -2009-05-28 another minor fix for Solaris boxes (antirez) -2009-05-28 minor fix for Solaris boxes (antirez) -2009-05-28 minor fix for Solaris boxes (antirez) -2009-05-27 maxmemory implemented (antirez) -2009-05-26 Redis git version modified to 0.101 in order to distinguish that from the latest tar.gz via INFO ;) (antirez) -2009-05-26 Redis 0.100 released (antirez) -2009-05-26 client libraries synched in git (antirez) -2009-05-26 ignore gcc warning about write() return code not checked. It is esplicitily this way since the "max number of clients reached" is a best-effort error (antirez) -2009-05-26 max bytes of a received command enlarged from 1k to 16k (antirez) -2009-05-26 RubyRedis: set TCP_NODELAY TCP socket option to to disable the neagle algorithm. Makes a huge difference under some OS, notably Linux (antirez) -2009-05-25 maxclients implemented, see redis.conf for details (antirez) -2009-05-25 INFO command now reports replication info (antirez) -2009-05-25 minor fix to RubyRedis about bulk commands sent without arguments (antirez) -2009-05-24 Warns if using the default config (antirez) -2009-05-24 Issue with redis-client used in scripts solved, now to check if the latest argument must come from standard input we do not check that stdin is or not a tty but the command arity (antirez) -2009-05-23 RubyRedis: now sets are returned as arrays again, and not as Set objects (antirez) -2009-05-23 SLAVEOF command documented (antirez) -2009-05-23 SLAVEOF command implemented for replication remote control (antirez) -2009-05-22 Fix: no connection timeout for the master! (antirez) -2009-05-22 replication slave timeout when receiving the initial bulk data set to 3600 seconds, now that replication is non-blocking the server must save the db before to start the async replication and this can take a lot of time with huge datasets (antirez) -2009-05-22 README tutorial now reflects the new proto (antirez) -2009-05-22 critical bug about glueoutputbuffers=yes fixed. Under load and with pipelining and clients disconnecting on the middle of the chat with the server, Redis could block. Now it's ok (antirez) -2009-05-22 TTL command doc added (antirez) -2009-05-22 TTL command implemented (antirez) -2009-05-22 S*STORE now return the cardinality of the resulting set (antirez) -2009-05-22 rubyredis more compatible with Redis-rb (antirez) -2009-05-21 minor indentation fix (antirez) -2009-05-21 timeout support and Redis-rb compatibility aliases implemented in RubyRedis (antirez) -2009-05-21 RubyRedis info postprocessor rewritten in a more functional way (antirez) -2009-05-21 dead code removed from RubyRedis (antirez) -2009-05-21 command postprocessing implemented into RubyRedis (antirez) -2009-05-20 Automagically reconnection of RubyRedis (antirez) -2009-05-20 RubyRedis: Array alike operators implemented (antirez) -2009-05-20 random testing code removed (antirez) -2009-05-20 RubyRedis DB selection forced at object creation (antirez) -2009-05-20 Initial version of an alternative Ruby client added (antirez) -2009-05-20 SDIFF / SDIFFSTORE added to doc (antirez) -2009-05-20 Aman Gupta changes merged (antirez) -2009-05-20 Merge git://github.com/tmm1/redis (antirez) -2009-05-19 Allow timeout=0 config to disable client timeouts (Aman Gupta) -2009-05-19 Partial qsort implemented in SORT command, only when both BY and LIMIT is used. minor fix for a warning compiling under Linux. (antirez) -2009-05-19 psort.c/h added. This is a partial qsort implementation that Redis will use when SORT+LIMIT is requested (antirez) -2009-05-17 Fix SINTER/UNIONSTORE to allow for &=/|= style operations (i.e. SINTERSTORE set1 set1 set2) (Aman Gupta) -2009-05-17 Optimize SDIFF to return as soon as the result set is empty (Aman Gupta) -2009-05-17 SDIFF/SDIFFSTORE implemnted unifying it with the implementation of SUNION/SUNIONSTORE (antirez) -2009-05-11 timestamp in log lines (antirez) -2009-05-11 Python client updated pushing from Ludo's repository (antirez) -2009-05-11 disconnect when we cannot read from the socket (Ludovico Magnocavallo) -2009-05-11 benchmark utility now supports random keys (antirez) -2009-05-10 minor doc changes (antirez) -2009-05-09 added tests for vararg DEL (antirez) -2009-05-09 DEL is now a vararg, IMPORTANT: memory leak fixed in loading DB code (antirez) -2009-05-09 doc changes (antirez) -2009-05-09 CPP client added thanks to Brian Hammond (antirez) -2009-05-06 Infinite number of arguments for MGET and all the other commands (antirez) -2009-05-04 Warns if /proc/sys/vm/overcommit_memory is set to 0 on Linux. Also make sure to don't resize the hash tables while the child process is saving in order to avoid copy-on-write of memory pages (antirez) -2009-04-30 zmalloc fix, return NULL or real malloc failure (antirez) -2009-04-30 more fixes for dict.c and the 150 million keys limit (antirez) -2009-04-30 dict.c modified to be able to handle more than 150,000,000 keys (antirez) -2009-04-29 fuzz stresser implemented in redis-test (antirez) -2009-04-29 fixed for HT resize check 32bits overflow (antirez) -2009-04-29 Check for fork() failure in background saving (antirez) -2009-04-29 fix for the LZF off-by-one bug added (antirez) -2009-04-28 print bytes used at exit on SHUTDOWN (antirez) -2009-04-28 SMOVE test added (antirez) -2009-04-28 SMOVE command implemented (antirez) -2009-04-28 less CPU usage in command parsing, case insensitive config directives (antirez) -2009-04-28 GETSET command doc added (antirez) -2009-04-28 GETSET tests (antirez) -2009-04-28 GETSET implemented (antirez) -2009-04-27 ability to specify a different file name for the DB (antirez) -2009-04-27 log file parsing code improved a bit (antirez) -2009-04-27 bgsave_in_progress field in INFO output (antirez) -2009-04-27 INCRBY/DECRBY now support 64bit increments, with tests (antirez) -2009-04-23 RANDOMKEY regression test added (antirez) -2009-04-23 dictGetRandomKey bug fixed, RANDOMKEY will not block the server anymore (antirez) -2009-04-22 FLUSHALL/FLUSHDB no longer sync on disk. Just increment the dirty counter by the number of elements removed, that will probably trigger a background saving operation (antirez) -2009-04-21 forgot to comment testing code in PHP lib. Now it is ok (antirez) -2009-04-21 PHP client ported to PHP5 and fixed (antirez) -2009-04-21 doc update (antirez) -2009-04-20 Non blocking replication (finally!). C-side linked lists API improved. (antirez) -2009-04-19 SUNION, SUNIONSTORE, Initial work on non blocking replication (antirez) -2009-04-10 Redis 0.091 released (antirez) -2009-04-10 SINTER/SINTERSTORE/SLEMENTS fix: misisng keys are now not errors, but just like empty sets (antirez) -2009-04-09 doc changes (antirez) -2009-04-08 TODO changes, minor change to default redis.conf (antirez) -2009-04-08 html doc updated (antirez) -2009-04-08 library clients update scripts (antirez) -2009-04-08 Ruby client updated (antirez) -2009-04-08 Lua client updated (antirez) -2009-04-08 Changelog updated (antirez) -2009-04-08 Merge git://github.com/ludoo/redis (antirez) -2009-04-08 add expire command to the php lib (Ludovico Magnocavallo) -2009-04-08 fix decode bug, add flush and info commands (Ludovico Magnocavallo) -2009-04-07 Rearrange redisObject struct to reduce memory usage in 64bit environments (as recommended http://groups.google.com/group/redis-db/msg/68f5a743f8f4e287) (Bob Potter) -2009-04-07 ruby19 compat: use each_line on string (Bob Potter) -2009-04-07 64bit fixes for usedmemory (Bob Potter) -2009-04-08 RANDOMKEY issue 26 fixed, generic test + regression added (antirez) -2009-04-06 Don't accept SAVE if BGSAVE is in progress (antirez) -2009-04-06 add expire command to the python lib (Ludovico Magnocavallo) -2009-04-03 persistent EXPIRE (antirez) -2009-04-03 dirty increment was missing in two points. TODO updated (antirez) -2009-04-02 LZF configured to initalize the HT in order to be determinsitic and play well with valgrind (antirez) -2009-04-02 fix select test (Ludovico Magnocavallo) -2009-04-02 fix trailing cr+nl in values (Ludovico Magnocavallo) -2009-04-02 compression/decompression of large values on disk now working (antirez) -2009-04-02 disable LZF compression since it's not able to load the DB for now, the load part is missing (antirez) -2009-04-02 new LZF files added (antirez) -2009-04-02 Fixed issue 23 about AUTH (antirez) -2009-04-02 Issue 22 fixed (antirez) -2009-04-01 non-lazy expired keys purging implemented (antirez) -2009-04-01 fastlz dependence removed (antirez) -2009-04-01 Initial implementation of EXPIRE (antirez) -2009-03-30 TODO updated (antirez) -2009-03-30 changelog added (antirez) -2009-03-28 redis-sha1 utility added (antirez) -2009-03-28 Integer encoding implemented in dump file. Doc updated (antirez) -2009-03-27 feature macros defined to play well with C99 (antirez) -2009-03-27 feature macros defined to play well with C99 (antirez) -2009-03-27 now Redis is C99-ok (antirez) -2009-03-27 IMPORTANT FIX: new dump format implementation was broken. Now it's ok but tests for the 32-bit case values are needed (antirez) -2009-03-27 ANSI-C compatibility changes (antirez) -2009-03-27 Ruby client library updated. Important changes in this new version! (antirez) -2009-03-26 Lua client added thanks to Daniele Alessandri (antirez) -2009-03-26 Lua client added thanks to Daniele Alessandri (antirez) -2009-03-26 AUTH merged from Brian Hammond fork, reworked a bit to fix minor problems (antirez) -2009-03-25 Adds AUTH command. (Brian Hammond) -2009-03-25 Nasty bug of the new DB format fixed, objects sharing implemented (antirez) -2009-03-25 doc update (antirez) -2009-03-25 Erlang client synched with Valentiono's repo (antirez) -2009-03-25 New file dump format, perl client library added (antirez) -2009-03-25 New protocol fix for LREM (antirez) -2009-03-24 two typos fixed (antirez) -2009-03-24 Now the Redis test uses the proper Tcl client library (antirez) -2009-03-24 Tcl client library (antirez) -2009-03-24 redis-benchmark sync with the new protocol (antirez) -2009-03-24 git mess :) (Ludovico Magnocavallo) -2009-03-24 sync python client to the new protocol (Ludovico Magnocavallo) -2009-03-24 protocol fix in SORT reply with null elements (antirez) -2009-03-24 protocol doc changed (antirez) -2009-03-24 Server replies now in the new format, test-redis.tcl and redis-cli modified accordingly (antirez) -2009-03-24 Python client library updated, thanks to Ludo! (antirez) -2009-03-24 random tested mode for test-redis.tcl, minor other stuff, version switched to 0.8 (antirez) -2009-03-23 Now MONITOR/SYNC cannot be issued multiple times (antirez) -2009-03-23 MONITOR command implemented. (antirez) -2009-03-23 lucsky changes imported. pid file path can now be configured, redis-cli fixes (antirez) -2009-03-23 Merge git://github.com/lucsky/redis (antirez) -2009-03-23 another missing free->zfree replacement fixed. Thanks to Ludo (antirez) -2009-03-23 Fixed redis-cli readLine loop to correctly handle EOF. (Luc Heinrich) -2009-03-23 Display the port on server startup. (Luc Heinrich) -2009-03-23 Allow to specify the pid file from the config file. (Luc Heinrich) -2009-03-23 Added gitignore file. (Luc Heinrich) -2009-03-22 MGET tests added (antirez) -2009-03-22 doc changes (antirez) -2009-03-22 added doc for MGET (antirez) -2009-03-22 redis-cli now checks the arity of vararg commnads (antirez) -2009-03-22 INFO fixed, MGET implemented, redis-cli implements INFO/MGET (antirez) -2009-03-22 first commit (antirez) \ No newline at end of file diff --git a/README b/README deleted file mode 100644 index 329eb1cb3..000000000 --- a/README +++ /dev/null @@ -1,154 +0,0 @@ -Where to find complete Redis documentation? -------------------------------------------- - -This README is just a fast "quick start" document. You can find more detailed -documentation at http://redis.io - -Building Redis --------------- - -Redis can be compiled and used on Linux, OSX, OpenBSD, NetBSD, FreeBSD. -We support big endian and little endian architectures. - -It may compile on Solaris derived systems (for instance SmartOS) but our -support for this platform is "best effort" and Redis is not guaranteed to -work as well as in Linux, OSX, and *BSD there. - -It is as simple as: - - % make - -You can run a 32 bit Redis binary using: - - % make 32bit - -After building Redis is a good idea to test it, using: - - % make test - -Fixing problems building 32 bit binaries ---------- - -If after building Redis with a 32 bit target you need to rebuild it -with a 64 bit target, or the other way around, you need to perform a -"make distclean" in the root directory of the Redis distribution. - -In case of build errors when trying to build a 32 bit binary of Redis, try -the following steps: - -* Install the packages libc6-dev-i386 (also try g++-multilib). -* Try using the following command line instead of "make 32bit": - - make CFLAGS="-m32 -march=native" LDFLAGS="-m32" - -Allocator ---------- - -Selecting a non-default memory allocator when building Redis is done by setting -the `MALLOC` environment variable. Redis is compiled and linked against libc -malloc by default, with the exception of jemalloc being the default on Linux -systems. This default was picked because jemalloc has proven to have fewer -fragmentation problems than libc malloc. - -To force compiling against libc malloc, use: - - % make MALLOC=libc - -To compile against jemalloc on Mac OS X systems, use: - - % make MALLOC=jemalloc - -Verbose build -------------- - -Redis will build with a user friendly colorized output by default. -If you want to see a more verbose output use the following: - - % make V=1 - -Running Redis -------------- - -To run Redis with the default configuration just type: - - % cd src - % ./redis-server - -If you want to provide your redis.conf, you have to run it using an additional -parameter (the path of the configuration file): - - % cd src - % ./redis-server /path/to/redis.conf - -It is possible to alter the Redis configuration passing parameters directly -as options using the command line. Examples: - - % ./redis-server --port 9999 --slaveof 127.0.0.1 6379 - % ./redis-server /etc/redis/6379.conf --loglevel debug - -All the options in redis.conf are also supported as options using the command -line, with exactly the same name. - -Playing with Redis ------------------- - -You can use redis-cli to play with Redis. Start a redis-server instance, -then in another terminal try the following: - - % cd src - % ./redis-cli - redis> ping - PONG - redis> set foo bar - OK - redis> get foo - "bar" - redis> incr mycounter - (integer) 1 - redis> incr mycounter - (integer) 2 - redis> - -You can find the list of all the available commands here: - - http://redis.io/commands - -Installing Redis ------------------ - -In order to install Redis binaries into /usr/local/bin just use: - - % make install - -You can use "make PREFIX=/some/other/directory install" if you wish to use a -different destination. - -Make install will just install binaries in your system, but will not configure -init scripts and configuration files in the appropriate place. This is not -needed if you want just to play a bit with Redis, but if you are installing -it the proper way for a production system, we have a script doing this -for Ubuntu and Debian systems: - - % cd utils - % ./install_server - -The script will ask you a few questions and will setup everything you need -to run Redis properly as a background daemon that will start again on -system reboots. - -You'll be able to stop and start Redis using the script named -/etc/init.d/redis_, for instance /etc/init.d/redis_6379. - -Code contributions ---- - -Note: by contributing code to the Redis project in any form, including sending -a pull request via Github, a code fragment or patch via private email or -public discussion groups, you agree to release your code under the terms -of the BSD license that you can find in the COPYING file included in the Redis -source distribution. - -Please see the CONTRIBUTING file in this source distribution for more -information. - -Enjoy! diff --git a/README.md b/README.md new file mode 100644 index 000000000..70a15790f --- /dev/null +++ b/README.md @@ -0,0 +1,446 @@ +This README is just a fast *quick start* document. You can find more detailed documentation at http://redis.io. + +What is Redis? +-------------- + +Redis is often referred as a *data structures* server. What this means is that Redis provides access to mutable data structures via a set of commands, which are sent using a *server-client* model with TCP sockets and a simple protocol. So different processes can query and modify the same data structures in a shared way. + +Data structures implemented into Redis have a few special properties: + +* Redis cares to store them on disk, even if they are always served and modified into the server memory. This means that Redis is fast, but that is also non-volatile. +* Implementation of data structures stress on memory efficiency, so data structures inside Redis will likely use less memory compared to the same data structure modeled using an high level programming language. +* Redis offers a number of features that are natural to find in a database, like replication, tunable levels of durability, cluster, high availability. + +Another good example is to think of Redis as a more complex version of memcached, where the operations are not just SETs and GETs, but operations to work with complex data types like Lists, Sets, ordered data structures, and so forth. + +If you want to know more, this is a list of selected starting points: + +* Introduction to Redis data types. http://redis.io/topics/data-types-intro +* Try Redis directly inside your browser. http://try.redis.io +* The full list of Redis commands. http://redis.io/commands +* There is much more inside the Redis official documentation. http://redis.io/documentation + +Building Redis +-------------- + +Redis can be compiled and used on Linux, OSX, OpenBSD, NetBSD, FreeBSD. +We support big endian and little endian architectures, and both 32 bit +and 64 bit systems. + +It may compile on Solaris derived systems (for instance SmartOS) but our +support for this platform is *best effort* and Redis is not guaranteed to +work as well as in Linux, OSX, and \*BSD there. + +It is as simple as: + + % make + +You can run a 32 bit Redis binary using: + + % make 32bit + +After building Redis, it is a good idea to test it using: + + % make test + +Fixing build problems with dependencies or cached build options +--------- + +Redis has some dependencies which are included into the `deps` directory. +`make` does not automatically rebuild dependencies even if something in +the source code of dependencies changes. + +When you update the source code with `git pull` or when code inside the +dependencies tree is modified in any other way, make sure to use the following +command in order to really clean everything and rebuild from scratch: + + make distclean + +This will clean: jemalloc, lua, hiredis, linenoise. + +Also if you force certain build options like 32bit target, no C compiler +optimizations (for debugging purposes), and other similar build time options, +those options are cached indefinitely until you issue a `make distclean` +command. + +Fixing problems building 32 bit binaries +--------- + +If after building Redis with a 32 bit target you need to rebuild it +with a 64 bit target, or the other way around, you need to perform a +`make distclean` in the root directory of the Redis distribution. + +In case of build errors when trying to build a 32 bit binary of Redis, try +the following steps: + +* Install the packages libc6-dev-i386 (also try g++-multilib). +* Try using the following command line instead of `make 32bit`: + `make CFLAGS="-m32 -march=native" LDFLAGS="-m32"` + +Allocator +--------- + +Selecting a non-default memory allocator when building Redis is done by setting +the `MALLOC` environment variable. Redis is compiled and linked against libc +malloc by default, with the exception of jemalloc being the default on Linux +systems. This default was picked because jemalloc has proven to have fewer +fragmentation problems than libc malloc. + +To force compiling against libc malloc, use: + + % make MALLOC=libc + +To compile against jemalloc on Mac OS X systems, use: + + % make MALLOC=jemalloc + +Verbose build +------------- + +Redis will build with a user friendly colorized output by default. +If you want to see a more verbose output use the following: + + % make V=1 + +Running Redis +------------- + +To run Redis with the default configuration just type: + + % cd src + % ./redis-server + +If you want to provide your redis.conf, you have to run it using an additional +parameter (the path of the configuration file): + + % cd src + % ./redis-server /path/to/redis.conf + +It is possible to alter the Redis configuration by passing parameters directly +as options using the command line. Examples: + + % ./redis-server --port 9999 --slaveof 127.0.0.1 6379 + % ./redis-server /etc/redis/6379.conf --loglevel debug + +All the options in redis.conf are also supported as options using the command +line, with exactly the same name. + +Playing with Redis +------------------ + +You can use redis-cli to play with Redis. Start a redis-server instance, +then in another terminal try the following: + + % cd src + % ./redis-cli + redis> ping + PONG + redis> set foo bar + OK + redis> get foo + "bar" + redis> incr mycounter + (integer) 1 + redis> incr mycounter + (integer) 2 + redis> + +You can find the list of all the available commands at http://redis.io/commands. + +Installing Redis +----------------- + +In order to install Redis binaries into /usr/local/bin just use: + + % make install + +You can use `make PREFIX=/some/other/directory install` if you wish to use a +different destination. + +Make install will just install binaries in your system, but will not configure +init scripts and configuration files in the appropriate place. This is not +needed if you want just to play a bit with Redis, but if you are installing +it the proper way for a production system, we have a script doing this +for Ubuntu and Debian systems: + + % cd utils + % ./install_server.sh + +The script will ask you a few questions and will setup everything you need +to run Redis properly as a background daemon that will start again on +system reboots. + +You'll be able to stop and start Redis using the script named +`/etc/init.d/redis_`, for instance `/etc/init.d/redis_6379`. + +Code contributions +----------------- + +Note: by contributing code to the Redis project in any form, including sending +a pull request via Github, a code fragment or patch via private email or +public discussion groups, you agree to release your code under the terms +of the BSD license that you can find in the [COPYING][1] file included in the Redis +source distribution. + +Please see the [CONTRIBUTING][2] file in this source distribution for more +information. + +[1]: https://github.com/antirez/redis/blob/unstable/COPYING +[2]: https://github.com/antirez/redis/blob/unstable/CONTRIBUTING + +Redis internals +=== + +If you are reading this README you are likely in front of a Github page +or you just untarred the Redis distribution tar ball. In both the cases +you are basically one step away from the source code, so here we explain +the Redis source code layout, what is in each file as a general idea, the +most important functions and structures inside the Redis server and so forth. +We keep all the discussion at a high level without digging into the details +since this document would be huge otherwise and our code base changes +continuously, but a general idea should be a good starting point to +understand more. Moreover most of the code is heavily commented and easy +to follow. + +Source code layout +--- + +The Redis root directory just contains this README, the Makefile which +calls the real Makefile inside the `src` directory and an example +configuration for Redis and Sentinel. You can find a few shell +scripts that are used in order to execute the Redis, Redis Cluster and +Redis Sentinel unit tests, which are implemented inside the `tests` +directory. + +Inside the root are the following important directories: + +* `src`: contains the Redis implementation, written in C. +* `tests`: contains the unit tests, implemented in Tcl. +* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `anitrez/redis`. An exception to this rule is `deps/geohash-int` which is the low level geocoding library used by Redis: it originated from a different project, but at this point it diverged so much that it is developed as a separated entity directly inside the Redis repository. + +There are a few more directories but they are not very important for our goals +here. We'll focus mostly on `src`, where the Redis implementation is contained, +exploring what there is inside each file. The order in which files are +exposed is the logical one to follow in order to disclose different layers +of complexity incrementally. + +Note: lately Redis was refactored quite a bit. Function names and file +names have been changed, so you may find that this documentation reflects the +`unstable` branch more closely. For instance in Redis 3.0 the `server.c` +and `server.h` files were named to `redis.c` and `redis.h`. However the overall +structure is the same. Keep in mind that all the new developments and pull +requests should be performed against the `unstable` branch. + +server.h +--- + +The simplest way to understand how a program works is to understand the +data structures it uses. So we'll start from the main header file of +Redis, which is `server.h`. + +All the server configuration and in general all the shared state is +defined in a global structure called `server`, of type `struct redisServer`. +A few important fields in this structure are: + +* `server.db` is an array of Redis databases, where data is stored. +* `server.commands` is the command table. +* `server.clients` is a linked list of clients connected to the server. +* `server.master` is a special client, the master, if the instance is a slave. + +There are tons of other fields. Most fields are commented directly inside +the structure definition. + +Another important Redis data structure is the one defining a client. +In the past it was called `redisClient`, now just `client`. The structure +has many fields, here we'll just show the main ones: + + struct client { + int fd; + sds querybuf; + int argc; + robj **argv; + redisDb *db; + int flags; + list *reply; + char buf[PROTO_REPLY_CHUNK_BYTES]; + ... many other fields ... + } + +The client structure defines a *connected client*: + +* The `fd` field is the client socket file descriptor. +* `argc` and `argv` are populated with the command the client is executing, so that functions implementing a given Redis command can read the arguments. +* `querybuf` accumulates the requests from the client, which are parsed by the Redis server according to the Redis protocol and executed by calling the implementations of the commands the client is executing. +* `reply` and `buf` are dynamic and static buffers that accumulate the replies the server sends to the client. These buffers are incrementally written to the socket as soon as the file descriptor is writable. + +As you can see in the client structure above, arguments in a command +are described as `robj` structures. The following is the full `robj` +structure, which defines a *Redis object*: + + typedef struct redisObject { + unsigned type:4; + unsigned encoding:4; + unsigned lru:LRU_BITS; /* lru time (relative to server.lruclock) */ + int refcount; + void *ptr; + } robj; + +Basically this structure can represent all the basic Redis data types like +strings, lists, sets, sorted sets and so forth. The interesting thing is that +it has a `type` field, so that it is possible to know what type a given +object has, and a `refcount`, so that the same object can be referenced +in multiple places without allocating it multiple times. Finally the `ptr` +field points to the actual representation of the object, which might vary +even for the same type, depending on the `encoding` used. + +Redis objects are used extensively in the Redis internals, however in order +to avoid the overhead of indirect accesses, recently in many places +we just use plain dynamic strings not wrapped inside a Redis object. + +server.c +--- + +This is the entry point of the Redis server, where the `main()` function +is defined. The following are the most important steps in order to startup +the Redis server. + +* `initServerConfig()` setups the default values of the `server` structure. +* `initServer()` allocates the data structures needed to operate, setup the listening socket, and so forth. +* `aeMain()` starts the event loop which listens for new connections. + +There are two special functions called periodically by the event loop: + +1. `serverCron()` is called periodically (according to `server.hz` frequency), and performs tasks that must be performed from time to time, like checking for timedout clients. +2. `beforeSleep()` is called every time the event loop fired, Redis served a few requests, and is returning back into the event loop. + +Inside server.c you can find code that handles other vital things of the Redis server: + +* `call()` is used in order to call a given command in the context of a given client. +* `activeExpireCycle()` handles eviciton of keys with a time to live set via the `EXPIRE` command. +* `freeMemoryIfNeeded()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive. +* The global variable `redisCommandTable` defines all the Redis commands, specifying the name of the command, the function implementing the command, the number of arguments required, and other properties of each command. + +networking.c +--- + +This file defines all the I/O functions with clients, masters and slaves +(which in Redis are just special clients): + +* `createClient()` allocates and initializes a new client. +* the `addReply*()` family of functions are used by commands implementations in order to append data to the client structure, that will be transmitted to the client as a reply for a given command executed. +* `writeToClient()` transmits the data pending in the output buffers to the client and is called by the *writable event handler* `sendReplyToClient()`. +* `readQueryFromClient()` is the *readable event handler* and accumulates data from read from the client into the query buffer. +* `processInputBuffer()` is the entry point in order to parse the client query buffer according to the Redis protocol. Once commands are ready to be processed, it calls `processCommand()` which is defined inside `server.c` in order to actually execute the command. +* `freeClient()` deallocates, disconnects and removes a client. + +aof.c and rdb.c +--- + +As you can guess from the names these files implement the RDB and AOF +persistence for Redis. Redis uses a persistence model based on the `fork()` +system call in order to create a thread with the same (shared) memory +content of the main Redis thread. This secondary thread dumps the content +of the memory on disk. This is used by `rdb.c` to create the snapshots +on disk and by `aof.c` in order to perform the AOF rewrite when the +append only file gets too big. + +The implementation inside `aof.c` has additional functions in order to +implement an API that allows commands to append new commands into the AOF +file as clients execute them. + +The `call()` function defined inside `server.c` is responsible to call +the functions that in turn will write the commands into the AOF. + +db.c +--- + +Certain Redis commands operate on specific data types, others are general. +Examples of generic commands are `DEL` and `EXPIRE`. They operate on keys +and not on their values specifically. All those generic commands are +defined inside `db.c`. + +Moreover `db.c` implements an API in order to perform certain operations +on the Redis dataset without directly accessing the internal data structures. + +The most important functions inside `db.c` which are used in many commands +implementations are the following: + +* `lookupKeyRead()` and `lookupKeyWrite()` are used in order to get a pointer to the value associated to a given key, or `NULL` if the key does not exist. +* `dbAdd()` and its higher level counterpart `setKey()` create a new key in a Redis database. +* `dbDelete()` removes a key and its associated value. +* `emptyDb()` removes an entire single database or all the databases defined. + +The rest of the file implements the generic commands exposed to the client. + +object.c +--- + +The `robj` structure defining Redis objects was already described. Inside +`object.c` there are all the functions that operate with Redis objects at +a basic level, like functions to allocate new objects, handle the reference +counting and so forth. Notable functions inside this file: + +* `incrRefcount()` and `decrRefCount()` are used in order to increment or decrement an object reference count. When it drops to 0 the object is finally freed. +* `createObject()` allocates a new object. There are also specialized functions to allocate string objects having a specific content, like `createStringObjectFromLongLong()` and similar functions. + +This file also implements the `OBJECT` command. + +replication.c +--- + +This is one of the most complex files inside Redis, it is recommended to +approach it only after getting a bit familiar with the rest of the code base. +In this file there is the implementation of both the master and slave role +of Redis. + +One of the most important functions inside this file is `replicationFeedSlaves()` that writes commands to the clients representing slave instances connected +to our master, so that the slaves can get the writes performed by the clients: +this way their data set will remain synchronized with the one in the master. + +This file also implements both the `SYNC` and `PSYNC` commands that are +used in order to perform the first synchronization between masters and +slaves, or to continue the replication after a disconnection. + +Other C files +--- + +* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c` and `t_zset.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types. +* `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand. +* `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information. +* `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel. +* `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally. +* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familar with the Lua API. +* `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3]. + +[3]: http://redis.io/topics/cluster-spec + +Anatomy of a Redis command +--- + +All the Redis commands are defined in the following way: + + void foobarCommand(client *c) { + printf("%s",c->argv[1]->ptr); /* Do something with the argument. */ + addReply(c,shared.ok); /* Reply something to the client. */ + } + +The command is then referenced inside `server.c` in the command table: + + {"foobar",foobarCommand,2,"rtF",0,NULL,0,0,0,0,0}, + +In the above example `2` is the number of arguments the command takes, +while `"rtF"` are the command flags, as documented in the command table +top comment inside `server.c`. + +After the command operates in some way, it returns a reply to the client, +usually using `addReply()` or a similar function defined inside `networking.c`. + +There are tons of commands implementations inside th Redis source code +that can serve as examples of actual commands implementations. To write +a few toy commands can be a good exercise to familiarize with the code base. + +There are also many other files not described here, but it is useless to +cover everything. We want to just help you with the first steps. +Eventually you'll find your way inside the Redis code base :-) + +Enjoy! diff --git a/deps/Makefile b/deps/Makefile index 5a95545de..e148a331c 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -58,12 +58,17 @@ ifeq ($(uname_S),SunOS) LUA_CFLAGS= -D__C99FEATURES__=1 endif -LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI $(CFLAGS) +LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS) LUA_LDFLAGS+= $(LDFLAGS) +# lua's Makefile defines AR="ar rcu", which is unusual, and makes it more +# challenging to cross-compile lua (and redis). These defines make it easier +# to fit redis into cross-compilation environments, which typically set AR. +AR=ar +ARFLAGS=rcu lua: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd lua/src && $(MAKE) all CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(LUA_LDFLAGS)" + cd lua/src && $(MAKE) all CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(LUA_LDFLAGS)" AR="$(AR) $(ARFLAGS)" .PHONY: lua @@ -72,7 +77,7 @@ JEMALLOC_LDFLAGS= $(LDFLAGS) jemalloc: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd jemalloc && ./configure --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" + cd jemalloc && ./configure --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a .PHONY: jemalloc diff --git a/deps/README.md b/deps/README.md new file mode 100644 index 000000000..0ce480046 --- /dev/null +++ b/deps/README.md @@ -0,0 +1,66 @@ +This directory contains all Redis dependencies, except for the libc that +should be provided by the operating system. + +* **Jemalloc** is our memory allocator, used as replacement for libc malloc on Linux by default. It has good performances and excellent fragmentation behavior. This component is upgraded from time to time. +* **geohash-int** is inside the dependencies directory but is actually part of the Redis project, since it is our private fork (heavily modified) of a library initially developed for Ardb, which is in turn a fork of Redis. +* **hiredis** is the official C client library for Redis. It is used by redis-cli, redis-benchmark and Redis Sentinel. It is part of the Redis official ecosystem but is developed externally from the Redis repository, so we just upgrade it as needed. +* **linenoise** is a readline replacement. It is developed by the same authors of Redis but is managed as a separated project and updated as needed. +* **lua** is Lua 5.1 with minor changes for security and additional libraries. + +How to upgrade the above dependencies +=== + +Jemalloc +--- + +Jemalloc is unmodified. We only change settings via the `configure` script of Jemalloc using the `--with-lg-quantum` option, setting it to the value of 3 instead of 4. This provides us with more size classes that better suit the Redis data structures, in order to gain memory efficiency. + +So in order to upgrade jemalloc: + +1. Remove the jemalloc directory. +2. Substitute it with the new jemalloc source tree. + +Geohash +--- + +This is never upgraded since it's part of the Redis project. If there are changes to merge from Ardb there is the need to manually check differences, but at this point the source code is pretty different. + +Hiredis +--- + +Hiredis uses the SDS string library, that must be the same version used inside Redis itself. Hiredis is also very critical for Sentinel. Historically Redis often used forked versions of hiredis in a way or the other. In order to upgrade it is adviced to take a lot of care: + +1. Check with diff if hiredis API changed and what impact it could have in Redis. +2. Make sure thet the SDS library inside Hiredis and inside Redis are compatible. +3. After the upgrade, run the Redis Sentinel test. +4. Check manually that redis-cli and redis-benchmark behave as expecteed, since we have no tests for CLI utilities currently. + +Linenoise +--- + +Linenoise is rarely upgraded as needed. The upgrade process is trivial since +Redis uses a non modified version of linenoise, so to upgrade just do the +following: + +1. Remove the linenoise directory. +2. Substitute it with the new linenoise source tree. + +Lua +--- + +We use Lua 5.1 and no upgrade is planned currently, since we don't want to break +Lua scripts for new Lua features: in the context of Redis Lua scripts the +capabilities of 5.1 are usually more than enough, the release is rock solid, +and we definitely don't want to break old scripts. + +So upgrading of Lua is up to the Redis project maintainers and should be a +manual procedure performed by taking a diff between the different versions. + +Currently we have at least the following differences between official Lua 5.1 +and our version: + +1. Makefile is modified to allow a different compiler than GCC. +2. We have the implementation source code, and directly link to the following external libraries: `lua_cjson.o`, `lua_struct.o`, `lua_cmsgpack.o` and `lua_bit.o`. +3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order toa void direct bytecode exectuion. + + diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index 1a4d60d28..c44b5c537 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -1,6 +1,7 @@ /hiredis-test -/hiredis-example* +/examples/hiredis-example* /*.o /*.so /*.dylib /*.a +/*.pc diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml new file mode 100644 index 000000000..ad08076d8 --- /dev/null +++ b/deps/hiredis/.travis.yml @@ -0,0 +1,39 @@ +language: c +sudo: false +compiler: + - gcc + - clang + +os: + - linux + - osx + +before_script: + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update; brew install redis; fi + +addons: + apt: + packages: + - libc6-dbg + - libc6-dev + - libc6:i386 + - libc6-dev-i386 + - libc6-dbg:i386 + - gcc-multilib + - valgrind + +env: + - CFLAGS="-Werror" + - PRE="valgrind --track-origins=yes --leak-check=full" + - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" + - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + +matrix: + exclude: + - os: osx + env: PRE="valgrind --track-origins=yes --leak-check=full" + + - os: osx + env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + +script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index d41db8a60..f92bcb3c9 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -1,3 +1,136 @@ +### 1.0.0 (unreleased) + +**Fixes**: + +* Catch a buffer overflow when formatting the error message +* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 +* Fix warnings, when compiled with -Wshadow +* Make hiredis compile in Cygwin on Windows, now CI-tested + +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. +If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + +* Remove backwards compatibility macro's + +This removes the following old function aliases, use the new name now: + +| Old | New | +| --------------------------- | ---------------------- | +| redisReplyReaderCreate | redisReaderCreate | +| redisReplyReaderCreate | redisReaderCreate | +| redisReplyReaderFree | redisReaderFree | +| redisReplyReaderFeed | redisReaderFeed | +| redisReplyReaderGetReply | redisReaderGetReply | +| redisReplyReaderSetPrivdata | redisReaderSetPrivdata | +| redisReplyReaderGetObject | redisReaderGetObject | +| redisReplyReaderGetError | redisReaderGetError | + +* The `DEBUG` variable in the Makefile was renamed to `DEBUG_FLAGS` + +Previously it broke some builds for people that had `DEBUG` set to some arbitrary value, +due to debugging other software. +By renaming we avoid unintentional name clashes. + +Simply rename `DEBUG` to `DEBUG_FLAGS` in your environment to make it working again. + +### 0.13.3 (2015-09-16) + +* Revert "Clear `REDIS_CONNECTED` flag when connection is closed". +* Make tests pass on FreeBSD (Thanks, Giacomo Olgeni) + + +If the `REDIS_CONNECTED` flag is cleared, +the async onDisconnect callback function will never be called. +This causes problems as the disconnect is never reported back to the user. + +### 0.13.2 (2015-08-25) + +* Prevent crash on pending replies in async code (Thanks, @switch-st) +* Clear `REDIS_CONNECTED` flag when connection is closed (Thanks, Jerry Jacobs) +* Add MacOS X addapter (Thanks, @dizzus) +* Add Qt adapter (Thanks, Pietro Cerutti) +* Add Ivykis adapter (Thanks, Gergely Nagy) + +All adapters are provided as is and are only tested where possible. + +### 0.13.1 (2015-05-03) + +This is a bug fix release. +The new `reconnect` method introduced new struct members, which clashed with pre-defined names in pre-C99 code. +Another commit forced C99 compilation just to make it work, but of course this is not desirable for outside projects. +Other non-C99 code can now use hiredis as usual again. +Sorry for the inconvenience. + +* Fix memory leak in async reply handling (Salvatore Sanfilippo) +* Rename struct member to avoid name clash with pre-c99 code (Alex Balashov, ncopa) + +### 0.13.0 (2015-04-16) + +This release adds a minimal Windows compatibility layer. +The parser, standalone since v0.12.0, can now be compiled on Windows +(and thus used in other client libraries as well) + +* Windows compatibility layer for parser code (tzickel) +* Properly escape data printed to PKGCONF file (Dan Skorupski) +* Fix tests when assert() undefined (Keith Bennett, Matt Stancliff) +* Implement a reconnect method for the client context, this changes the structure of `redisContext` (Aaron Bedra) + +### 0.12.1 (2015-01-26) + +* Fix `make install`: DESTDIR support, install all required files, install PKGCONF in proper location +* Fix `make test` as 32 bit build on 64 bit platform + +### 0.12.0 (2015-01-22) + +* Add optional KeepAlive support + +* Try again on EINTR errors + +* Add libuv adapter + +* Add IPv6 support + +* Remove possiblity of multiple close on same fd + +* Add ability to bind source address on connect + +* Add redisConnectFd() and redisFreeKeepFd() + +* Fix getaddrinfo() memory leak + +* Free string if it is unused (fixes memory leak) + +* Improve redisAppendCommandArgv performance 2.5x + +* Add support for SO_REUSEADDR + +* Fix redisvFormatCommand format parsing + +* Add GLib 2.0 adapter + +* Refactor reading code into read.c + +* Fix errno error buffers to not clobber errors + +* Generate pkgconf during build + +* Silence _BSD_SOURCE warnings + +* Improve digit counting for multibulk creation + + +### 0.11.0 + +* Increase the maximum multi-bulk reply depth to 7. + +* Increase the read buffer size from 2k to 16k. + +* Use poll(2) instead of select(2) to support large fds (>= 1024). + ### 0.10.1 * Makefile overhaul. Important to check out if you override one or more diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 16b8767b1..9a4de8360 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,24 +3,50 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o -BINS=hiredis-example hiredis-test +OBJ=net.o hiredis.o sds.o async.o read.o +EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +TESTS=hiredis-test LIBNAME=libhiredis +PKGCONFNAME=hiredis.pc -HIREDIS_MAJOR=0 -HIREDIS_MINOR=10 +HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') +HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') +HIREDIS_PATCH=$(shell grep HIREDIS_PATCH hiredis.h | awk '{print $$3}') +HIREDIS_SONAME=$(shell grep HIREDIS_SONAME hiredis.h | awk '{print $$3}') + +# Installation related variables and target +PREFIX?=/usr/local +INCLUDE_PATH?=include/hiredis +LIBRARY_PATH?=lib +PKGCONF_PATH?=pkgconfig +INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH) +INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH) +INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH) + +# redis-server configuration used for testing +REDIS_PORT=56379 +REDIS_SERVER=redis-server +define REDIS_TEST_CONFIG + daemonize yes + pidfile /tmp/hiredis-test-redis.pid + port $(REDIS_PORT) + bind 127.0.0.1 + unixsocket /tmp/hiredis-test-redis.sock +endef +export REDIS_TEST_CONFIG # Fallback to gcc when $CC is not in $PATH. CC:=$(shell sh -c 'type $(CC) >/dev/null 2>/dev/null && echo $(CC) || echo gcc') +CXX:=$(shell sh -c 'type $(CXX) >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -DEBUG?= -g -ggdb -REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG) $(ARCH) +DEBUG_FLAGS?= -g -ggdb +REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) $(ARCH) REAL_LDFLAGS=$(LDFLAGS) $(ARCH) DYLIBSUFFIX=so STLIBSUFFIX=a -DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR).$(HIREDIS_MINOR) +DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) @@ -36,20 +62,20 @@ ifeq ($(uname_S),SunOS) endif ifeq ($(uname_S),Darwin) DYLIBSUFFIX=dylib - DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(DYLIBSUFFIX) - DYLIB_MAJOR_NAME=$(LIBNAME).$(HIREDIS_MAJOR).$(DYLIBSUFFIX) + DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX) DYLIB_MAKE_CMD=$(CC) -shared -Wl,-install_name,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) endif -all: $(DYLIBNAME) $(BINS) +all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) # Deps (use make dep to generate this) -net.o: net.c fmacros.h net.h hiredis.h -async.o: async.c async.h hiredis.h sds.h dict.c dict.h -example.o: example.c hiredis.h -hiredis.o: hiredis.c fmacros.h hiredis.h net.h sds.h +async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h +dict.o: dict.c fmacros.h dict.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +read.o: read.c fmacros.h read.h sds.h sds.o: sds.c sds.h -test.o: test.c hiredis.h +test.o: test.c fmacros.h hiredis.h read.h sds.h $(DYLIBNAME): $(OBJ) $(DYLIB_MAKE_CMD) $(OBJ) @@ -61,36 +87,68 @@ dynamic: $(DYLIBNAME) static: $(STLIBNAME) # Binaries: -hiredis-example-libevent: example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o $@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -levent example-libevent.c $(STLIBNAME) +hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) -hiredis-example-libev: example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o $@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -lev example-libev.c $(STLIBNAME) +hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + +hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) $(shell pkg-config --cflags --libs glib-2.0) -I. $< $(STLIBNAME) + +hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + +hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) ifndef AE_DIR hiredis-example-ae: @echo "Please specify AE_DIR (e.g. /src)" @false else -hiredis-example-ae: example-ae.c adapters/ae.h $(STLIBNAME) - $(CC) -o $@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I$(AE_DIR) $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o example-ae.c $(STLIBNAME) +hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(AE_DIR) $< $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o $(AE_DIR)/../deps/jemalloc/lib/libjemalloc.a -pthread $(STLIBNAME) endif +ifndef LIBUV_DIR +hiredis-example-libuv: + @echo "Please specify LIBUV_DIR (e.g. ../libuv/)" + @false +else +hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) +endif + +ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) +hiredis-example-qt: + @echo "Please specify QT_MOC, QT_INCLUDE_DIR AND QT_LIBRARY_DIR" + @false +else +hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) + $(QT_MOC) adapters/qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \ + $(CXX) -x c++ -o qt-adapter-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore + $(QT_MOC) examples/example-qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \ + $(CXX) -x c++ -o qt-example-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore + $(CXX) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore -L$(QT_LIBRARY_DIR) qt-adapter-moc.o qt-example-moc.o $< -pthread $(STLIBNAME) -lQtCore +endif + +hiredis-example: examples/example.c $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + +examples: $(EXAMPLES) + +hiredis-test: test.o $(STLIBNAME) + hiredis-%: %.o $(STLIBNAME) - $(CC) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) test: hiredis-test ./hiredis-test check: hiredis-test - echo \ - "daemonize yes\n" \ - "pidfile /tmp/hiredis-test-redis.pid\n" \ - "port 56379\n" \ - "bind 127.0.0.1\n" \ - "unixsocket /tmp/hiredis-test-redis.sock" \ - | redis-server - - ./hiredis-test -h 127.0.0.1 -p 56379 -s /tmp/hiredis-test-redis.sock || \ + @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - + $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ ( kill `cat /tmp/hiredis-test-redis.pid` && false ) kill `cat /tmp/hiredis-test-redis.pid` @@ -98,31 +156,38 @@ check: hiredis-test $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(BINS) hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: $(CC) -MM *.c -# Installation related variables and target -PREFIX?=/usr/local -INCLUDE_PATH?=include/hiredis -LIBRARY_PATH?=lib -INSTALL_INCLUDE_PATH= $(PREFIX)/$(INCLUDE_PATH) -INSTALL_LIBRARY_PATH= $(PREFIX)/$(LIBRARY_PATH) - ifeq ($(uname_S),SunOS) INSTALL?= cp -r endif INSTALL?= cp -a -install: $(DYLIBNAME) $(STLIBNAME) +$(PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis >> $@ + @echo Description: Minimalistic C client library for Redis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Libs: -L\$${libdir} -lhiredis >> $@ + @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ + +install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) - $(INSTALL) hiredis.h async.h adapters $(INSTALL_INCLUDE_PATH) + $(INSTALL) hiredis.h async.h read.h sds.h adapters $(INSTALL_INCLUDE_PATH) $(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME) - cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIB_MAJOR_NAME) - cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MAJOR_NAME) $(DYLIBNAME) + cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) $(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH) + mkdir -p $(INSTALL_PKGCONF_PATH) + $(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH) 32bit: @echo "" @@ -130,6 +195,10 @@ install: $(DYLIBNAME) $(STLIBNAME) @echo "" $(MAKE) CFLAGS="-m32" LDFLAGS="-m32" +32bit-vars: + $(eval CFLAGS=-m32) + $(eval LDFLAGS=-m32) + gprof: $(MAKE) CFLAGS="-pg" LDFLAGS="-pg" @@ -145,4 +214,4 @@ coverage: gcov noopt: $(MAKE) OPTIMIZATION="" -.PHONY: all test check clean dep install 32bit gprof gcov noopt +.PHONY: all test check clean dep install 32bit 32bit-vars gprof gcov noopt diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index 62fe1067b..01223ea59 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -1,9 +1,13 @@ +[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis) + +**This Readme reflects the latest changed in the master branch. See [v0.13.3](https://github.com/redis/hiredis/tree/v0.13.3) for the Readme and documentation for the latest release.** + # HIREDIS Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database. It is minimalistic because it just adds minimal support for the protocol, but -at the same time it uses an high level printf-alike API in order to make it +at the same time it uses a high level printf-alike API in order to make it much higher level than otherwise suggested by its minimal code base and the lack of explicit bindings for every Redis command. @@ -18,7 +22,15 @@ Redis version >= 1.2.0. The library comes with multiple APIs. There is the *synchronous API*, the *asynchronous API* and the *reply parsing API*. -## UPGRADING +## Upgrading to `1.0.0` + +Version 1.0.0 marks a stable release of hiredis. +It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory. +It also bundles the updated `sds` library, to sync up with upstream and Redis. +For most applications a recompile against the new hiredis should be enough. +For code changes see the [Changelog](CHANGELOG.md). + +## Upgrading from `<0.9.0` Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing code using hiredis should not be a big pain. The key thing to keep in mind when @@ -29,51 +41,62 @@ the stateless 0.0.1 that only has a file descriptor to work with. To consume the synchronous API, there are only a few function calls that need to be introduced: - redisContext *redisConnect(const char *ip, int port); - void *redisCommand(redisContext *c, const char *format, ...); - void freeReplyObject(void *reply); +```c +redisContext *redisConnect(const char *ip, int port); +void *redisCommand(redisContext *c, const char *format, ...); +void freeReplyObject(void *reply); +``` ### Connecting The function `redisConnect` is used to create a so-called `redisContext`. The context is where Hiredis holds state for a connection. The `redisContext` -struct has an integer `err` field that is non-zero when an the connection is in +struct has an integer `err` field that is non-zero when the connection is in an error state. The field `errstr` will contain a string with a description of the error. More information on errors can be found in the **Errors** section. After trying to connect to Redis using `redisConnect` you should check the `err` field to see if establishing the connection was successful: - - redisContext *c = redisConnect("127.0.0.1", 6379); - if (c->err) { +```c +redisContext *c = redisConnect("127.0.0.1", 6379); +if (c == NULL || c->err) { + if (c) { printf("Error: %s\n", c->errstr); // handle error + } else { + printf("Can't allocate redis context\n"); } +} +``` + +*Note: A `redisContext` is not thread-safe.* ### Sending commands There are several ways to issue commands to Redis. The first that will be introduced is `redisCommand`. This function takes a format similar to printf. In the simplest form, it is used like this: - - reply = redisCommand(context, "SET foo bar"); +```c +reply = redisCommand(context, "SET foo bar"); +``` The specifier `%s` interpolates a string in the command, and uses `strlen` to determine the length of the string: - - reply = redisCommand(context, "SET foo %s", value); - +```c +reply = redisCommand(context, "SET foo %s", value); +``` When you need to pass binary safe strings in a command, the `%b` specifier can be used. Together with a pointer to the string, it requires a `size_t` length argument of the string: - - reply = redisCommand(context, "SET foo %b", value, valuelen); - +```c +reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen); +``` Internally, Hiredis splits the command in different arguments and will convert it to the protocol used to communicate with Redis. One or more spaces separates arguments, so you can use the specifiers anywhere in an argument: - - reply = redisCommand(context, "SET key:%s %s", myid, value); +```c +reply = redisCommand(context, "SET key:%s %s", myid, value); +``` ### Using replies @@ -112,11 +135,11 @@ was received: Redis may reply with nested arrays but this is fully supported. Replies should be freed using the `freeReplyObject()` function. -Note that this function will take care of freeing sub-replies objects +Note that this function will take care of freeing sub-reply objects contained in arrays and nested arrays, so there is no need for the user to free the sub replies (it is actually harmful and will corrupt the memory). -**Important:** the current version of hiredis (0.10.0) free's replies when the +**Important:** the current version of hiredis (0.10.0) frees replies when the asynchronous API is used. This means you should not call `freeReplyObject` when you use this API. The reply is cleaned up by hiredis _after_ the callback returns. This behavior will probably change in future releases, so make sure to @@ -125,19 +148,19 @@ keep an eye on the changelog when upgrading (see issue #39). ### Cleaning up To disconnect and free the context the following function can be used: - - void redisFree(redisContext *c); - -This function immediately closes the socket and then free's the allocations done in +```c +void redisFree(redisContext *c); +``` +This function immediately closes the socket and then frees the allocations done in creating the context. ### Sending commands (cont'd) Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands. It has the following prototype: - - void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); - +```c +void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); +``` It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments @@ -167,10 +190,10 @@ The function `redisGetReply` is exported as part of the Hiredis API and can be u is expected on the socket. To pipeline commands, the only things that needs to be done is filling up the output buffer. For this cause, two commands can be used that are identical to the `redisCommand` family, apart from not returning a reply: - - void redisAppendCommand(redisContext *c, const char *format, ...); - void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); - +```c +void redisAppendCommand(redisContext *c, const char *format, ...); +void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); +``` After calling either function one or more times, `redisGetReply` can be used to receive the subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where the latter means an error occurred while reading a reply. Just as with the other commands, @@ -178,24 +201,24 @@ the `err` field in the context can be used to find out what the cause of this er The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and a single call to `read(2)`): - - redisReply *reply; - redisAppendCommand(context,"SET foo bar"); - redisAppendCommand(context,"GET foo"); - redisGetReply(context,&reply); // reply for SET - freeReplyObject(reply); - redisGetReply(context,&reply); // reply for GET - freeReplyObject(reply); - +```c +redisReply *reply; +redisAppendCommand(context,"SET foo bar"); +redisAppendCommand(context,"GET foo"); +redisGetReply(context,&reply); // reply for SET +freeReplyObject(reply); +redisGetReply(context,&reply); // reply for GET +freeReplyObject(reply); +``` This API can also be used to implement a blocking subscriber: - - reply = redisCommand(context,"SUBSCRIBE foo"); +```c +reply = redisCommand(context,"SUBSCRIBE foo"); +freeReplyObject(reply); +while(redisGetReply(context,&reply) == REDIS_OK) { + // consume message freeReplyObject(reply); - while(redisGetReply(context,&reply) == REDIS_OK) { - // consume message - freeReplyObject(reply); - } - +} +``` ### Errors When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is @@ -235,58 +258,62 @@ should be checked after creation to see if there were errors creating the connec Because the connection that will be created is non-blocking, the kernel is not able to instantly return if the specified host and port is able to accept a connection. - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); - if (c->err) { - printf("Error: %s\n", c->errstr); - // handle error - } +*Note: A `redisAsyncContext` is not thread-safe.* + +```c +redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); +if (c->err) { + printf("Error: %s\n", c->errstr); + // handle error +} +``` The asynchronous context can hold a disconnect callback function that is called when the connection is disconnected (either because of an error or per user request). This function should have the following prototype: - - void(const redisAsyncContext *c, int status); - +```c +void(const redisAsyncContext *c, int status); +``` On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err` field in the context can be accessed to find out the cause of the error. -The context object is always free'd after the disconnect callback fired. When a reconnect is needed, +The context object is always freed after the disconnect callback fired. When a reconnect is needed, the disconnect callback is a good point to do so. Setting the disconnect callback can only be done once per context. For subsequent calls it will return `REDIS_ERR`. The function to set the disconnect callback has the following prototype: - - int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); - +```c +int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); +``` ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. Therefore, unlike the synchronous API, there is only a single way to send commands. Because commands are sent to Redis asynchronously, issuing a command requires a callback function that is called when the reply is received. Reply callbacks should have the following prototype: - - void(redisAsyncContext *c, void *reply, void *privdata); - +```c +void(redisAsyncContext *c, void *reply, void *privdata); +``` The `privdata` argument can be used to curry arbitrary data to the callback from the point where the command is initially queued for execution. The functions that can be used to issue commands in an asynchronous context are: - - int redisAsyncCommand( - redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, - const char *format, ...); - int redisAsyncCommandArgv( - redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, - int argc, const char **argv, const size_t *argvlen); - +```c +int redisAsyncCommand( + redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, + const char *format, ...); +int redisAsyncCommandArgv( + redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, + int argc, const char **argv, const size_t *argvlen); +``` Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is returned on calls to the `redisAsyncCommand` family. -If the reply for a command with a `NULL` callback is read, it is immediately free'd. When the callback -for a command is non-`NULL`, the memory is free'd immediately following the callback: the reply is only +If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback +for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only valid for the duration of the callback. All pending callbacks are called with a `NULL` reply when the context encountered an error. @@ -294,14 +321,14 @@ All pending callbacks are called with a `NULL` reply when the context encountere ### Disconnecting An asynchronous connection can be terminated using: - - void redisAsyncDisconnect(redisAsyncContext *ac); - +```c +void redisAsyncDisconnect(redisAsyncContext *ac); +``` When this function is called, the connection is **not** immediately terminated. Instead, new commands are no longer accepted and the connection is only terminated when all pending commands have been written to the socket, their respective replies have been read and their respective callbacks have been executed. After this, the disconnection callback is executed with the -`REDIS_OK` status and the context object is free'd. +`REDIS_OK` status and the context object is freed. ### Hooking it up to event library *X* @@ -314,12 +341,12 @@ Hiredis comes with a reply parsing API that makes it easy for writing higher level language bindings. The reply parsing API consists of the following functions: - - redisReader *redisReaderCreate(void); - void redisReaderFree(redisReader *reader); - int redisReaderFeed(redisReader *reader, const char *buf, size_t len); - int redisReaderGetReply(redisReader *reader, void **reply); - +```c +redisReader *redisReaderCreate(void); +void redisReaderFree(redisReader *reader); +int redisReaderFeed(redisReader *reader, const char *buf, size_t len); +int redisReaderGetReply(redisReader *reader, void **reply); +``` The same set of functions are used internally by hiredis when creating a normal Redis context, the above API just exposes it to the user for a direct usage. @@ -337,6 +364,9 @@ and a reply object (as described above) via `void **reply`. The returned status can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went wrong (either a protocol error, or an out of memory error). +The parser limits the level of nesting for multi bulk payloads to 7. If the +multi bulk nesting level is higher than this, the parser returns an error. + ### Customizing replies The function `redisReaderGetReply` creates `redisReply` and makes the function @@ -356,7 +386,7 @@ Both when using the Reader API directly or when using it indirectly via a normal Redis context, the redisReader structure uses a buffer in order to accumulate data from the server. Usually this buffer is destroyed when it is empty and is larger than 16 -kb in order to avoid wasting memory in unused buffers +KiB in order to avoid wasting memory in unused buffers However when working with very big payloads destroying the buffer may slow down performances considerably, so it is possible to modify the max size of @@ -366,9 +396,9 @@ value for an idle buffer, so the buffer will never get freed. For instance if you have a normal Redis context you can set the maximum idle buffer to zero (unlimited) just with: - - context->reader->maxbuf = 0; - +```c +context->reader->maxbuf = 0; +``` This should be done only in order to maximize performances when working with large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again as soon as possible in order to prevent allocation of useless memory. @@ -376,4 +406,6 @@ as soon as possible in order to prevent allocation of useless memory. ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and +Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/deps/hiredis/adapters/glib.h b/deps/hiredis/adapters/glib.h new file mode 100644 index 000000000..e0a6411d3 --- /dev/null +++ b/deps/hiredis/adapters/glib.h @@ -0,0 +1,153 @@ +#ifndef __HIREDIS_GLIB_H__ +#define __HIREDIS_GLIB_H__ + +#include + +#include "../hiredis.h" +#include "../async.h" + +typedef struct +{ + GSource source; + redisAsyncContext *ac; + GPollFD poll_fd; +} RedisSource; + +static void +redis_source_add_read (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events |= G_IO_IN; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_del_read (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events &= ~G_IO_IN; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_add_write (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events |= G_IO_OUT; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_del_write (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + g_return_if_fail(source); + source->poll_fd.events &= ~G_IO_OUT; + g_main_context_wakeup(g_source_get_context((GSource *)data)); +} + +static void +redis_source_cleanup (gpointer data) +{ + RedisSource *source = (RedisSource *)data; + + g_return_if_fail(source); + + redis_source_del_read(source); + redis_source_del_write(source); + /* + * It is not our responsibility to remove ourself from the + * current main loop. However, we will remove the GPollFD. + */ + if (source->poll_fd.fd >= 0) { + g_source_remove_poll((GSource *)data, &source->poll_fd); + source->poll_fd.fd = -1; + } +} + +static gboolean +redis_source_prepare (GSource *source, + gint *timeout_) +{ + RedisSource *redis = (RedisSource *)source; + *timeout_ = -1; + return !!(redis->poll_fd.events & redis->poll_fd.revents); +} + +static gboolean +redis_source_check (GSource *source) +{ + RedisSource *redis = (RedisSource *)source; + return !!(redis->poll_fd.events & redis->poll_fd.revents); +} + +static gboolean +redis_source_dispatch (GSource *source, + GSourceFunc callback, + gpointer user_data) +{ + RedisSource *redis = (RedisSource *)source; + + if ((redis->poll_fd.revents & G_IO_OUT)) { + redisAsyncHandleWrite(redis->ac); + redis->poll_fd.revents &= ~G_IO_OUT; + } + + if ((redis->poll_fd.revents & G_IO_IN)) { + redisAsyncHandleRead(redis->ac); + redis->poll_fd.revents &= ~G_IO_IN; + } + + if (callback) { + return callback(user_data); + } + + return TRUE; +} + +static void +redis_source_finalize (GSource *source) +{ + RedisSource *redis = (RedisSource *)source; + + if (redis->poll_fd.fd >= 0) { + g_source_remove_poll(source, &redis->poll_fd); + redis->poll_fd.fd = -1; + } +} + +static GSource * +redis_source_new (redisAsyncContext *ac) +{ + static GSourceFuncs source_funcs = { + .prepare = redis_source_prepare, + .check = redis_source_check, + .dispatch = redis_source_dispatch, + .finalize = redis_source_finalize, + }; + redisContext *c = &ac->c; + RedisSource *source; + + g_return_val_if_fail(ac != NULL, NULL); + + source = (RedisSource *)g_source_new(&source_funcs, sizeof *source); + source->ac = ac; + source->poll_fd.fd = c->fd; + source->poll_fd.events = 0; + source->poll_fd.revents = 0; + g_source_add_poll((GSource *)source, &source->poll_fd); + + ac->ev.addRead = redis_source_add_read; + ac->ev.delRead = redis_source_del_read; + ac->ev.addWrite = redis_source_add_write; + ac->ev.delWrite = redis_source_del_write; + ac->ev.cleanup = redis_source_cleanup; + ac->ev.data = source; + + return (GSource *)source; +} + +#endif /* __HIREDIS_GLIB_H__ */ diff --git a/deps/hiredis/adapters/ivykis.h b/deps/hiredis/adapters/ivykis.h new file mode 100644 index 000000000..6a12a868a --- /dev/null +++ b/deps/hiredis/adapters/ivykis.h @@ -0,0 +1,81 @@ +#ifndef __HIREDIS_IVYKIS_H__ +#define __HIREDIS_IVYKIS_H__ +#include +#include "../hiredis.h" +#include "../async.h" + +typedef struct redisIvykisEvents { + redisAsyncContext *context; + struct iv_fd fd; +} redisIvykisEvents; + +static void redisIvykisReadEvent(void *arg) { + redisAsyncContext *context = (redisAsyncContext *)arg; + redisAsyncHandleRead(context); +} + +static void redisIvykisWriteEvent(void *arg) { + redisAsyncContext *context = (redisAsyncContext *)arg; + redisAsyncHandleWrite(context); +} + +static void redisIvykisAddRead(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_in(&e->fd, redisIvykisReadEvent); +} + +static void redisIvykisDelRead(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_in(&e->fd, NULL); +} + +static void redisIvykisAddWrite(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_out(&e->fd, redisIvykisWriteEvent); +} + +static void redisIvykisDelWrite(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + iv_fd_set_handler_out(&e->fd, NULL); +} + +static void redisIvykisCleanup(void *privdata) { + redisIvykisEvents *e = (redisIvykisEvents*)privdata; + + iv_fd_unregister(&e->fd); + free(e); +} + +static int redisIvykisAttach(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisIvykisEvents *e; + + /* Nothing should be attached when something is already attached */ + if (ac->ev.data != NULL) + return REDIS_ERR; + + /* Create container for context and r/w events */ + e = (redisIvykisEvents*)malloc(sizeof(*e)); + e->context = ac; + + /* Register functions to start/stop listening for events */ + ac->ev.addRead = redisIvykisAddRead; + ac->ev.delRead = redisIvykisDelRead; + ac->ev.addWrite = redisIvykisAddWrite; + ac->ev.delWrite = redisIvykisDelWrite; + ac->ev.cleanup = redisIvykisCleanup; + ac->ev.data = e; + + /* Initialize and install read/write events */ + IV_FD_INIT(&e->fd); + e->fd.fd = c->fd; + e->fd.handler_in = redisIvykisReadEvent; + e->fd.handler_out = redisIvykisWriteEvent; + e->fd.handler_err = NULL; + e->fd.cookie = e->context; + + iv_fd_register(&e->fd); + + return REDIS_OK; +} +#endif diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 1c2b271bb..273d8b2dd 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -30,13 +30,13 @@ #ifndef __HIREDIS_LIBEVENT_H__ #define __HIREDIS_LIBEVENT_H__ -#include +#include #include "../hiredis.h" #include "../async.h" typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event rev, wev; + struct event *rev, *wev; } redisLibeventEvents; static void redisLibeventReadEvent(int fd, short event, void *arg) { @@ -53,28 +53,28 @@ static void redisLibeventWriteEvent(int fd, short event, void *arg) { static void redisLibeventAddRead(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(&e->rev,NULL); + event_add(e->rev,NULL); } static void redisLibeventDelRead(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(&e->rev); + event_del(e->rev); } static void redisLibeventAddWrite(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(&e->wev,NULL); + event_add(e->wev,NULL); } static void redisLibeventDelWrite(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(&e->wev); + event_del(e->wev); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(&e->rev); - event_del(&e->wev); + event_del(e->rev); + event_del(e->wev); free(e); } @@ -99,10 +99,10 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.data = e; /* Initialize and install read/write events */ - event_set(&e->rev,c->fd,EV_READ,redisLibeventReadEvent,e); - event_set(&e->wev,c->fd,EV_WRITE,redisLibeventWriteEvent,e); - event_base_set(base,&e->rev); - event_base_set(base,&e->wev); + e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); + e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); + event_add(e->rev, NULL); + event_add(e->wev, NULL); return REDIS_OK; } #endif diff --git a/deps/hiredis/adapters/libuv.h b/deps/hiredis/adapters/libuv.h new file mode 100644 index 000000000..ff08c25e1 --- /dev/null +++ b/deps/hiredis/adapters/libuv.h @@ -0,0 +1,122 @@ +#ifndef __HIREDIS_LIBUV_H__ +#define __HIREDIS_LIBUV_H__ +#include +#include +#include "../hiredis.h" +#include "../async.h" +#include + +typedef struct redisLibuvEvents { + redisAsyncContext* context; + uv_poll_t handle; + int events; +} redisLibuvEvents; + + +static void redisLibuvPoll(uv_poll_t* handle, int status, int events) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + + if (status != 0) { + return; + } + + if (p->context != NULL && (events & UV_READABLE)) { + redisAsyncHandleRead(p->context); + } + if (p->context != NULL && (events & UV_WRITABLE)) { + redisAsyncHandleWrite(p->context); + } +} + + +static void redisLibuvAddRead(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events |= UV_READABLE; + + uv_poll_start(&p->handle, p->events, redisLibuvPoll); +} + + +static void redisLibuvDelRead(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events &= ~UV_READABLE; + + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } +} + + +static void redisLibuvAddWrite(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events |= UV_WRITABLE; + + uv_poll_start(&p->handle, p->events, redisLibuvPoll); +} + + +static void redisLibuvDelWrite(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events &= ~UV_WRITABLE; + + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } +} + + +static void on_close(uv_handle_t* handle) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + + free(p); +} + + +static void redisLibuvCleanup(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->context = NULL; // indicate that context might no longer exist + uv_close((uv_handle_t*)&p->handle, on_close); +} + + +static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { + redisContext *c = &(ac->c); + + if (ac->ev.data != NULL) { + return REDIS_ERR; + } + + ac->ev.addRead = redisLibuvAddRead; + ac->ev.delRead = redisLibuvDelRead; + ac->ev.addWrite = redisLibuvAddWrite; + ac->ev.delWrite = redisLibuvDelWrite; + ac->ev.cleanup = redisLibuvCleanup; + + redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p)); + + if (!p) { + return REDIS_ERR; + } + + memset(p, 0, sizeof(*p)); + + if (uv_poll_init(loop, &p->handle, c->fd) != 0) { + return REDIS_ERR; + } + + ac->ev.data = p; + p->handle.data = p; + p->context = ac; + + return REDIS_OK; +} +#endif diff --git a/deps/hiredis/adapters/macosx.h b/deps/hiredis/adapters/macosx.h new file mode 100644 index 000000000..72121f606 --- /dev/null +++ b/deps/hiredis/adapters/macosx.h @@ -0,0 +1,114 @@ +// +// Created by Дмитрий Бахвалов on 13.07.15. +// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved. +// + +#ifndef __HIREDIS_MACOSX_H__ +#define __HIREDIS_MACOSX_H__ + +#include + +#include "../hiredis.h" +#include "../async.h" + +typedef struct { + redisAsyncContext *context; + CFSocketRef socketRef; + CFRunLoopSourceRef sourceRef; +} RedisRunLoop; + +static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) { + if( redisRunLoop != NULL ) { + if( redisRunLoop->sourceRef != NULL ) { + CFRunLoopSourceInvalidate(redisRunLoop->sourceRef); + CFRelease(redisRunLoop->sourceRef); + } + if( redisRunLoop->socketRef != NULL ) { + CFSocketInvalidate(redisRunLoop->socketRef); + CFRelease(redisRunLoop->socketRef); + } + free(redisRunLoop); + } + return REDIS_ERR; +} + +static void redisMacOSAddRead(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack); +} + +static void redisMacOSDelRead(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack); +} + +static void redisMacOSAddWrite(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack); +} + +static void redisMacOSDelWrite(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack); +} + +static void redisMacOSCleanup(void *privdata) { + RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata; + freeRedisRunLoop(redisRunLoop); +} + +static void redisMacOSAsyncCallback(CFSocketRef __unused s, CFSocketCallBackType callbackType, CFDataRef __unused address, const void __unused *data, void *info) { + redisAsyncContext* context = (redisAsyncContext*) info; + + switch (callbackType) { + case kCFSocketReadCallBack: + redisAsyncHandleRead(context); + break; + + case kCFSocketWriteCallBack: + redisAsyncHandleWrite(context); + break; + + default: + break; + } +} + +static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLoop) { + redisContext *redisCtx = &(redisAsyncCtx->c); + + /* Nothing should be attached when something is already attached */ + if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR; + + RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop)); + if( !redisRunLoop ) return REDIS_ERR; + + /* Setup redis stuff */ + redisRunLoop->context = redisAsyncCtx; + + redisAsyncCtx->ev.addRead = redisMacOSAddRead; + redisAsyncCtx->ev.delRead = redisMacOSDelRead; + redisAsyncCtx->ev.addWrite = redisMacOSAddWrite; + redisAsyncCtx->ev.delWrite = redisMacOSDelWrite; + redisAsyncCtx->ev.cleanup = redisMacOSCleanup; + redisAsyncCtx->ev.data = redisRunLoop; + + /* Initialize and install read/write events */ + CFSocketContext socketCtx = { 0, redisAsyncCtx, NULL, NULL, NULL }; + + redisRunLoop->socketRef = CFSocketCreateWithNative(NULL, redisCtx->fd, + kCFSocketReadCallBack | kCFSocketWriteCallBack, + redisMacOSAsyncCallback, + &socketCtx); + if( !redisRunLoop->socketRef ) return freeRedisRunLoop(redisRunLoop); + + redisRunLoop->sourceRef = CFSocketCreateRunLoopSource(NULL, redisRunLoop->socketRef, 0); + if( !redisRunLoop->sourceRef ) return freeRedisRunLoop(redisRunLoop); + + CFRunLoopAddSource(runLoop, redisRunLoop->sourceRef, kCFRunLoopDefaultMode); + + return REDIS_OK; +} + +#endif + diff --git a/deps/hiredis/adapters/qt.h b/deps/hiredis/adapters/qt.h new file mode 100644 index 000000000..5cc02e6ce --- /dev/null +++ b/deps/hiredis/adapters/qt.h @@ -0,0 +1,135 @@ +/*- + * Copyright (C) 2014 Pietro Cerutti + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef __HIREDIS_QT_H__ +#define __HIREDIS_QT_H__ +#include +#include "../async.h" + +static void RedisQtAddRead(void *); +static void RedisQtDelRead(void *); +static void RedisQtAddWrite(void *); +static void RedisQtDelWrite(void *); +static void RedisQtCleanup(void *); + +class RedisQtAdapter : public QObject { + + Q_OBJECT + + friend + void RedisQtAddRead(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->addRead(); + } + + friend + void RedisQtDelRead(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->delRead(); + } + + friend + void RedisQtAddWrite(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->addWrite(); + } + + friend + void RedisQtDelWrite(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->delWrite(); + } + + friend + void RedisQtCleanup(void * adapter) { + RedisQtAdapter * a = static_cast(adapter); + a->cleanup(); + } + + public: + RedisQtAdapter(QObject * parent = 0) + : QObject(parent), m_ctx(0), m_read(0), m_write(0) { } + + ~RedisQtAdapter() { + if (m_ctx != 0) { + m_ctx->ev.data = NULL; + } + } + + int setContext(redisAsyncContext * ac) { + if (ac->ev.data != NULL) { + return REDIS_ERR; + } + m_ctx = ac; + m_ctx->ev.data = this; + m_ctx->ev.addRead = RedisQtAddRead; + m_ctx->ev.delRead = RedisQtDelRead; + m_ctx->ev.addWrite = RedisQtAddWrite; + m_ctx->ev.delWrite = RedisQtDelWrite; + m_ctx->ev.cleanup = RedisQtCleanup; + return REDIS_OK; + } + + private: + void addRead() { + if (m_read) return; + m_read = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Read, 0); + connect(m_read, SIGNAL(activated(int)), this, SLOT(read())); + } + + void delRead() { + if (!m_read) return; + delete m_read; + m_read = 0; + } + + void addWrite() { + if (m_write) return; + m_write = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Write, 0); + connect(m_write, SIGNAL(activated(int)), this, SLOT(write())); + } + + void delWrite() { + if (!m_write) return; + delete m_write; + m_write = 0; + } + + void cleanup() { + delRead(); + delWrite(); + } + + private slots: + void read() { redisAsyncHandleRead(m_ctx); } + void write() { redisAsyncHandleWrite(m_ctx); } + + private: + redisAsyncContext * m_ctx; + QSocketNotifier * m_read; + QSocketNotifier * m_write; +}; + +#endif /* !__HIREDIS_QT_H__ */ diff --git a/deps/hiredis/appveyor.yml b/deps/hiredis/appveyor.yml new file mode 100644 index 000000000..06bbef117 --- /dev/null +++ b/deps/hiredis/appveyor.yml @@ -0,0 +1,36 @@ +# Appveyor configuration file for CI build of hiredis on Windows (under Cygwin) +environment: + matrix: + - CYG_ROOT: C:\cygwin64 + CYG_SETUP: setup-x86_64.exe + CYG_MIRROR: http://cygwin.mirror.constant.com + CYG_CACHE: C:\cygwin64\var\cache\setup + CYG_BASH: C:\cygwin64\bin\bash + CC: gcc + - CYG_ROOT: C:\cygwin + CYG_SETUP: setup-x86.exe + CYG_MIRROR: http://cygwin.mirror.constant.com + CYG_CACHE: C:\cygwin\var\cache\setup + CYG_BASH: C:\cygwin\bin\bash + CC: gcc + TARGET: 32bit + TARGET_VARS: 32bit-vars + +# Cache Cygwin files to speed up build +cache: + - '%CYG_CACHE%' +clone_depth: 1 + +# Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail +init: + - git config --global core.autocrlf input + +# Install needed build dependencies +install: + - ps: 'Start-FileDownload "http://cygwin.com/$env:CYG_SETUP" -FileName "$env:CYG_SETUP"' + - '%CYG_SETUP% --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages automake,bison,gcc-core,libtool,make,gettext-devel,gettext,intltool,pkg-config,clang,llvm > NUL 2>&1' + - '%CYG_BASH% -lc "cygcheck -dc cygwin"' + +build_script: + - 'echo building...' + - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0c); /* The regular connect functions will always set the flag REDIS_CONNECTED. @@ -136,21 +142,62 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { /* We want the error field to be accessible directly instead of requiring * an indirection to the redisContext struct. */ static void __redisAsyncCopyError(redisAsyncContext *ac) { + if (!ac) + return; + redisContext *c = &(ac->c); ac->err = c->err; ac->errstr = c->errstr; } redisAsyncContext *redisAsyncConnect(const char *ip, int port) { - redisContext *c = redisConnectNonBlock(ip,port); + redisContext *c; + redisAsyncContext *ac; + + c = redisConnectNonBlock(ip,port); + if (c == NULL) + return NULL; + + ac = redisAsyncInitialize(c); + if (ac == NULL) { + redisFree(c); + return NULL; + } + + __redisAsyncCopyError(ac); + return ac; +} + +redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); + redisAsyncContext *ac = redisAsyncInitialize(c); + __redisAsyncCopyError(ac); + return ac; +} + +redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); redisAsyncContext *ac = redisAsyncInitialize(c); __redisAsyncCopyError(ac); return ac; } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c = redisConnectUnixNonBlock(path); - redisAsyncContext *ac = redisAsyncInitialize(c); + redisContext *c; + redisAsyncContext *ac; + + c = redisConnectUnixNonBlock(path); + if (c == NULL) + return NULL; + + ac = redisAsyncInitialize(c); + if (ac == NULL) { + redisFree(c); + return NULL; + } + __redisAsyncCopyError(ac); return ac; } @@ -182,6 +229,9 @@ static int __redisPushCallback(redisCallbackList *list, redisCallback *source) { /* Copy callback from stack to heap */ cb = malloc(sizeof(*cb)); + if (cb == NULL) + return REDIS_ERR_OOM; + if (source != NULL) { memcpy(cb,source,sizeof(*cb)); cb->next = NULL; @@ -360,7 +410,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, void redisProcessCallbacks(redisAsyncContext *ac) { redisContext *c = &(ac->c); - redisCallback cb; + redisCallback cb = {NULL, NULL, NULL}; void *reply = NULL; int status; @@ -368,11 +418,12 @@ void redisProcessCallbacks(redisAsyncContext *ac) { if (reply == NULL) { /* When the connection is being disconnected and there are * no more replies, this is the cue to really disconnect. */ - if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0) { + if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0 + && ac->replies.head == NULL) { __redisAsyncDisconnect(ac); return; } - + /* If monitor mode, repush callback */ if(c->flags & REDIS_MONITORING) { __redisPushCallback(&ac->replies,&cb); @@ -404,6 +455,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) { if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) { c->err = REDIS_ERR_OTHER; snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str); + c->reader->fn->freeObject(reply); __redisAsyncDisconnect(ac); return; } @@ -437,12 +489,12 @@ void redisProcessCallbacks(redisAsyncContext *ac) { } /* Internal helper function to detect socket status the first time a read or - * write event fires. When connecting was not succesful, the connect callback + * write event fires. When connecting was not successful, the connect callback * is called with a REDIS_ERR status and the context is free'd. */ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); - if (redisCheckSocketError(c,c->fd) == REDIS_ERR) { + if (redisCheckSocketError(c) == REDIS_ERR) { /* Try again later when connect(2) is still in progress. */ if (errno == EINPROGRESS) return REDIS_OK; @@ -511,8 +563,8 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { /* Sets a pointer to the first argument and its length starting at p. Returns * the number of bytes to skip to get to the following argument. */ -static char *nextArgument(char *start, char **str, size_t *len) { - char *p = start; +static const char *nextArgument(const char *start, const char **str, size_t *len) { + const char *p = start; if (p[0] != '$') { p = strchr(p,'$'); if (p == NULL) return NULL; @@ -528,14 +580,15 @@ static char *nextArgument(char *start, char **str, size_t *len) { /* Helper function for the redisAsyncCommand* family of functions. Writes a * formatted command to the output buffer and registers the provided callback * function with the context. */ -static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, char *cmd, size_t len) { +static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { redisContext *c = &(ac->c); redisCallback cb; int pvariant, hasnext; - char *cstr, *astr; + const char *cstr, *astr; size_t clen, alen; - char *p; + const char *p; sds sname; + int ret; /* Don't accept new commands when the connection is about to be closed. */ if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR; @@ -559,9 +612,11 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void while ((p = nextArgument(p,&astr,&alen)) != NULL) { sname = sdsnewlen(astr,alen); if (pvariant) - dictReplace(ac->sub.patterns,sname,&cb); + ret = dictReplace(ac->sub.patterns,sname,&cb); else - dictReplace(ac->sub.channels,sname,&cb); + ret = dictReplace(ac->sub.channels,sname,&cb); + + if (ret == 0) sdsfree(sname); } } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) { /* It is only useful to call (P)UNSUBSCRIBE when the context is @@ -597,6 +652,11 @@ int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdat int len; int status; len = redisvFormatCommand(&cmd,format,ap); + + /* We don't want to pass -1 or -2 to future functions as a length. */ + if (len < 0) + return REDIS_ERR; + status = __redisAsyncCommand(ac,fn,privdata,cmd,len); free(cmd); return status; @@ -612,11 +672,16 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata } int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) { - char *cmd; + sds cmd; int len; int status; - len = redisFormatCommandArgv(&cmd,argc,argv,argvlen); + len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - free(cmd); + sdsfree(cmd); + return status; +} + +int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { + int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 268274e8e..59cbf469b 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -102,6 +102,9 @@ typedef struct redisAsyncContext { /* Functions that proxy to hiredis */ redisAsyncContext *redisAsyncConnect(const char *ip, int port); +redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); +redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, + const char *source_addr); redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); @@ -117,6 +120,7 @@ void redisAsyncHandleWrite(redisAsyncContext *ac); int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap); int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...); int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen); +int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len); #ifdef __cplusplus } diff --git a/deps/hiredis/dict.c b/deps/hiredis/dict.c index 79b1041ca..e17a62546 100644 --- a/deps/hiredis/dict.c +++ b/deps/hiredis/dict.c @@ -161,7 +161,7 @@ static int dictReplace(dict *ht, void *key, void *val) { dictEntry *entry, auxentry; /* Try to add the element. If the key - * does not exists dictAdd will suceed. */ + * does not exists dictAdd will succeed. */ if (dictAdd(ht, key, val) == DICT_OK) return 1; /* It already exists, get the entry */ @@ -293,7 +293,7 @@ static void dictReleaseIterator(dictIterator *iter) { /* Expand the hash table if needed */ static int _dictExpandIfNeeded(dict *ht) { - /* If the hash table is empty expand it to the intial size, + /* If the hash table is empty expand it to the initial size, * if the table is "full" dobule its size. */ if (ht->size == 0) return dictExpand(ht, DICT_HT_INITIAL_SIZE); diff --git a/deps/hiredis/example-ae.c b/deps/hiredis/examples/example-ae.c similarity index 89% rename from deps/hiredis/example-ae.c rename to deps/hiredis/examples/example-ae.c index 5ed34a3a6..8efa7306a 100644 --- a/deps/hiredis/example-ae.c +++ b/deps/hiredis/examples/example-ae.c @@ -2,9 +2,10 @@ #include #include #include -#include "hiredis.h" -#include "async.h" -#include "adapters/ae.h" + +#include +#include +#include /* Put event loop in the global scope, so it can be explicitly stopped */ static aeEventLoop *loop; @@ -21,17 +22,22 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { void connectCallback(const redisAsyncContext *c, int status) { if (status != REDIS_OK) { printf("Error: %s\n", c->errstr); + aeStop(loop); return; } + printf("Connected...\n"); } void disconnectCallback(const redisAsyncContext *c, int status) { if (status != REDIS_OK) { printf("Error: %s\n", c->errstr); + aeStop(loop); return; } + printf("Disconnected...\n"); + aeStop(loop); } int main (int argc, char **argv) { @@ -44,7 +50,7 @@ int main (int argc, char **argv) { return 1; } - loop = aeCreateEventLoop(); + loop = aeCreateEventLoop(64); redisAeAttach(loop, c); redisAsyncSetConnectCallback(c,connectCallback); redisAsyncSetDisconnectCallback(c,disconnectCallback); diff --git a/deps/hiredis/examples/example-glib.c b/deps/hiredis/examples/example-glib.c new file mode 100644 index 000000000..d6e10f8e8 --- /dev/null +++ b/deps/hiredis/examples/example-glib.c @@ -0,0 +1,73 @@ +#include + +#include +#include +#include + +static GMainLoop *mainloop; + +static void +connect_cb (const redisAsyncContext *ac G_GNUC_UNUSED, + int status) +{ + if (status != REDIS_OK) { + g_printerr("Failed to connect: %s\n", ac->errstr); + g_main_loop_quit(mainloop); + } else { + g_printerr("Connected...\n"); + } +} + +static void +disconnect_cb (const redisAsyncContext *ac G_GNUC_UNUSED, + int status) +{ + if (status != REDIS_OK) { + g_error("Failed to disconnect: %s", ac->errstr); + } else { + g_printerr("Disconnected...\n"); + g_main_loop_quit(mainloop); + } +} + +static void +command_cb(redisAsyncContext *ac, + gpointer r, + gpointer user_data G_GNUC_UNUSED) +{ + redisReply *reply = r; + + if (reply) { + g_print("REPLY: %s\n", reply->str); + } + + redisAsyncDisconnect(ac); +} + +gint +main (gint argc G_GNUC_UNUSED, + gchar *argv[] G_GNUC_UNUSED) +{ + redisAsyncContext *ac; + GMainContext *context = NULL; + GSource *source; + + ac = redisAsyncConnect("127.0.0.1", 6379); + if (ac->err) { + g_printerr("%s\n", ac->errstr); + exit(EXIT_FAILURE); + } + + source = redis_source_new(ac); + mainloop = g_main_loop_new(context, FALSE); + g_source_attach(source, context); + + redisAsyncSetConnectCallback(ac, connect_cb); + redisAsyncSetDisconnectCallback(ac, disconnect_cb); + redisAsyncCommand(ac, command_cb, NULL, "SET key 1234"); + redisAsyncCommand(ac, command_cb, NULL, "GET key"); + + g_main_loop_run(mainloop); + + return EXIT_SUCCESS; +} diff --git a/deps/hiredis/examples/example-ivykis.c b/deps/hiredis/examples/example-ivykis.c new file mode 100644 index 000000000..67affcef3 --- /dev/null +++ b/deps/hiredis/examples/example-ivykis.c @@ -0,0 +1,58 @@ +#include +#include +#include +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + iv_init(); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisIvykisAttach(c); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + + iv_main(); + + iv_deinit(); + + return 0; +} diff --git a/deps/hiredis/example-libev.c b/deps/hiredis/examples/example-libev.c similarity index 95% rename from deps/hiredis/example-libev.c rename to deps/hiredis/examples/example-libev.c index 7894f1f48..cc8b166ec 100644 --- a/deps/hiredis/example-libev.c +++ b/deps/hiredis/examples/example-libev.c @@ -2,9 +2,10 @@ #include #include #include -#include "hiredis.h" -#include "async.h" -#include "adapters/libev.h" + +#include +#include +#include void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; diff --git a/deps/hiredis/example-libevent.c b/deps/hiredis/examples/example-libevent.c similarity index 95% rename from deps/hiredis/example-libevent.c rename to deps/hiredis/examples/example-libevent.c index 9da8e02bf..d333c22b7 100644 --- a/deps/hiredis/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -2,9 +2,10 @@ #include #include #include -#include "hiredis.h" -#include "async.h" -#include "adapters/libevent.h" + +#include +#include +#include void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; diff --git a/deps/hiredis/examples/example-libuv.c b/deps/hiredis/examples/example-libuv.c new file mode 100644 index 000000000..a5462d410 --- /dev/null +++ b/deps/hiredis/examples/example-libuv.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + uv_loop_t* loop = uv_default_loop(); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisLibuvAttach(c,loop); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + uv_run(loop, UV_RUN_DEFAULT); + return 0; +} diff --git a/deps/hiredis/examples/example-macosx.c b/deps/hiredis/examples/example-macosx.c new file mode 100644 index 000000000..bc84ed5ba --- /dev/null +++ b/deps/hiredis/examples/example-macosx.c @@ -0,0 +1,66 @@ +// +// Created by Дмитрий Бахвалов on 13.07.15. +// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved. +// + +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + CFRunLoopStop(CFRunLoopGetCurrent()); + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + CFRunLoopRef loop = CFRunLoopGetCurrent(); + if( !loop ) { + printf("Error: Cannot get current run loop\n"); + return 1; + } + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisMacOSAttach(c, loop); + + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + + CFRunLoopRun(); + + return 0; +} + diff --git a/deps/hiredis/examples/example-qt.cpp b/deps/hiredis/examples/example-qt.cpp new file mode 100644 index 000000000..f524c3f3d --- /dev/null +++ b/deps/hiredis/examples/example-qt.cpp @@ -0,0 +1,46 @@ +#include +using namespace std; + +#include +#include + +#include "example-qt.h" + +void getCallback(redisAsyncContext *, void * r, void * privdata) { + + redisReply * reply = static_cast(r); + ExampleQt * ex = static_cast(privdata); + if (reply == nullptr || ex == nullptr) return; + + cout << "key: " << reply->str << endl; + + ex->finish(); +} + +void ExampleQt::run() { + + m_ctx = redisAsyncConnect("localhost", 6379); + + if (m_ctx->err) { + cerr << "Error: " << m_ctx->errstr << endl; + redisAsyncFree(m_ctx); + emit finished(); + } + + m_adapter.setContext(m_ctx); + + redisAsyncCommand(m_ctx, NULL, NULL, "SET key %s", m_value); + redisAsyncCommand(m_ctx, getCallback, this, "GET key"); +} + +int main (int argc, char **argv) { + + QCoreApplication app(argc, argv); + + ExampleQt example(argv[argc-1]); + + QObject::connect(&example, SIGNAL(finished()), &app, SLOT(quit())); + QTimer::singleShot(0, &example, SLOT(run())); + + return app.exec(); +} diff --git a/deps/hiredis/examples/example-qt.h b/deps/hiredis/examples/example-qt.h new file mode 100644 index 000000000..374f47666 --- /dev/null +++ b/deps/hiredis/examples/example-qt.h @@ -0,0 +1,32 @@ +#ifndef __HIREDIS_EXAMPLE_QT_H +#define __HIREDIS_EXAMPLE_QT_H + +#include + +class ExampleQt : public QObject { + + Q_OBJECT + + public: + ExampleQt(const char * value, QObject * parent = 0) + : QObject(parent), m_value(value) {} + + signals: + void finished(); + + public slots: + void run(); + + private: + void finish() { emit finished(); } + + private: + const char * m_value; + redisAsyncContext * m_ctx; + RedisQtAdapter m_adapter; + + friend + void getCallback(redisAsyncContext *, void *, void *); +}; + +#endif /* !__HIREDIS_EXAMPLE_QT_H */ diff --git a/deps/hiredis/example.c b/deps/hiredis/examples/example.c similarity index 71% rename from deps/hiredis/example.c rename to deps/hiredis/examples/example.c index 90ff9ed5e..4d494c55a 100644 --- a/deps/hiredis/example.c +++ b/deps/hiredis/examples/example.c @@ -2,17 +2,24 @@ #include #include -#include "hiredis.h" +#include -int main(void) { +int main(int argc, char **argv) { unsigned int j; redisContext *c; redisReply *reply; + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout((char*)"127.0.0.2", 6379, timeout); - if (c->err) { - printf("Connection error: %s\n", c->errstr); + c = redisConnectWithTimeout(hostname, port, timeout); + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } exit(1); } @@ -27,7 +34,7 @@ int main(void) { freeReplyObject(reply); /* Set a key using binary safe API */ - reply = redisCommand(c,"SET %b %b", "bar", 3, "hello", 5); + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); printf("SET (binary API): %s\n", reply->str); freeReplyObject(reply); @@ -50,7 +57,7 @@ int main(void) { for (j = 0; j < 10; j++) { char buf[64]; - snprintf(buf,64,"%d",j); + snprintf(buf,64,"%u",j); reply = redisCommand(c,"LPUSH mylist element-%s", buf); freeReplyObject(reply); } @@ -64,5 +71,8 @@ int main(void) { } freeReplyObject(reply); + /* Disconnects and frees the context */ + redisFree(c); + return 0; } diff --git a/deps/hiredis/fmacros.h b/deps/hiredis/fmacros.h index 96324eb49..14fed6060 100644 --- a/deps/hiredis/fmacros.h +++ b/deps/hiredis/fmacros.h @@ -1,16 +1,25 @@ #ifndef __HIREDIS_FMACRO_H #define __HIREDIS_FMACRO_H -#if !defined(_BSD_SOURCE) +#if defined(__linux__) #define _BSD_SOURCE +#define _DEFAULT_SOURCE +#endif + +#if defined(__CYGWIN__) +#include #endif #if defined(__sun__) #define _POSIX_C_SOURCE 200112L -#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) -#define _XOPEN_SOURCE 600 #else -#define _XOPEN_SOURCE +#if !(defined(__APPLE__) && defined(__MACH__)) +#define _XOPEN_SOURCE 600 +#endif +#endif + +#if defined(__APPLE__) && defined(__MACH__) +#define _OSX #endif #endif diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 4709ee325..18bdfc99c 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -1,6 +1,8 @@ /* * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger * * All rights reserved. * @@ -73,6 +75,9 @@ void freeReplyObject(void *reply) { redisReply *r = reply; size_t j; + if (r == NULL) + return; + switch(r->type) { case REDIS_REPLY_INTEGER: break; /* Nothing to free */ @@ -183,504 +188,23 @@ static void *createNilObject(const redisReadTask *task) { return r; } -static void __redisReaderSetError(redisReader *r, int type, const char *str) { - size_t len; - - if (r->reply != NULL && r->fn && r->fn->freeObject) { - r->fn->freeObject(r->reply); - r->reply = NULL; - } - - /* Clear input buffer on errors. */ - if (r->buf != NULL) { - sdsfree(r->buf); - r->buf = NULL; - r->pos = r->len = 0; - } - - /* Reset task stack. */ - r->ridx = -1; - - /* Set error. */ - r->err = type; - len = strlen(str); - len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1); - memcpy(r->errstr,str,len); - r->errstr[len] = '\0'; -} - -static size_t chrtos(char *buf, size_t size, char byte) { - size_t len = 0; - - switch(byte) { - case '\\': - case '"': - len = snprintf(buf,size,"\"\\%c\"",byte); - break; - case '\n': len = snprintf(buf,size,"\"\\n\""); break; - case '\r': len = snprintf(buf,size,"\"\\r\""); break; - case '\t': len = snprintf(buf,size,"\"\\t\""); break; - case '\a': len = snprintf(buf,size,"\"\\a\""); break; - case '\b': len = snprintf(buf,size,"\"\\b\""); break; - default: - if (isprint(byte)) - len = snprintf(buf,size,"\"%c\"",byte); - else - len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte); - break; - } - - return len; -} - -static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) { - char cbuf[8], sbuf[128]; - - chrtos(cbuf,sizeof(cbuf),byte); - snprintf(sbuf,sizeof(sbuf), - "Protocol error, got %s as reply type byte", cbuf); - __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf); -} - -static void __redisReaderSetErrorOOM(redisReader *r) { - __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory"); -} - -static char *readBytes(redisReader *r, unsigned int bytes) { - char *p; - if (r->len-r->pos >= bytes) { - p = r->buf+r->pos; - r->pos += bytes; - return p; - } - return NULL; -} - -/* Find pointer to \r\n. */ -static char *seekNewline(char *s, size_t len) { - int pos = 0; - int _len = len-1; - - /* Position should be < len-1 because the character at "pos" should be - * followed by a \n. Note that strchr cannot be used because it doesn't - * allow to search a limited length and the buffer that is being searched - * might not have a trailing NULL character. */ - while (pos < _len) { - while(pos < _len && s[pos] != '\r') pos++; - if (s[pos] != '\r') { - /* Not found. */ - return NULL; - } else { - if (s[pos+1] == '\n') { - /* Found. */ - return s+pos; - } else { - /* Continue searching. */ - pos++; - } - } - } - return NULL; -} - -/* Read a long long value starting at *s, under the assumption that it will be - * terminated by \r\n. Ambiguously returns -1 for unexpected input. */ -static long long readLongLong(char *s) { - long long v = 0; - int dec, mult = 1; - char c; - - if (*s == '-') { - mult = -1; - s++; - } else if (*s == '+') { - mult = 1; - s++; - } - - while ((c = *(s++)) != '\r') { - dec = c - '0'; - if (dec >= 0 && dec < 10) { - v *= 10; - v += dec; - } else { - /* Should not happen... */ - return -1; - } - } - - return mult*v; -} - -static char *readLine(redisReader *r, int *_len) { - char *p, *s; - int len; - - p = r->buf+r->pos; - s = seekNewline(p,(r->len-r->pos)); - if (s != NULL) { - len = s-(r->buf+r->pos); - r->pos += len+2; /* skip \r\n */ - if (_len) *_len = len; - return p; - } - return NULL; -} - -static void moveToNextTask(redisReader *r) { - redisReadTask *cur, *prv; - while (r->ridx >= 0) { - /* Return a.s.a.p. when the stack is now empty. */ - if (r->ridx == 0) { - r->ridx--; - return; - } - - cur = &(r->rstack[r->ridx]); - prv = &(r->rstack[r->ridx-1]); - assert(prv->type == REDIS_REPLY_ARRAY); - if (cur->idx == prv->elements-1) { - r->ridx--; - } else { - /* Reset the type because the next item can be anything */ - assert(cur->idx < prv->elements); - cur->type = -1; - cur->elements = -1; - cur->idx++; - return; - } - } -} - -static int processLineItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - void *obj; - char *p; - int len; - - if ((p = readLine(r,&len)) != NULL) { - if (cur->type == REDIS_REPLY_INTEGER) { - if (r->fn && r->fn->createInteger) - obj = r->fn->createInteger(cur,readLongLong(p)); - else - obj = (void*)REDIS_REPLY_INTEGER; - } else { - /* Type will be error or status. */ - if (r->fn && r->fn->createString) - obj = r->fn->createString(cur,p,len); - else - obj = (void*)(size_t)(cur->type); - } - - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - /* Set reply if this is the root object. */ - if (r->ridx == 0) r->reply = obj; - moveToNextTask(r); - return REDIS_OK; - } - - return REDIS_ERR; -} - -static int processBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - void *obj = NULL; - char *p, *s; - long len; - unsigned long bytelen; - int success = 0; - - p = r->buf+r->pos; - s = seekNewline(p,r->len-r->pos); - if (s != NULL) { - p = r->buf+r->pos; - bytelen = s-(r->buf+r->pos)+2; /* include \r\n */ - len = readLongLong(p); - - if (len < 0) { - /* The nil object can always be created. */ - if (r->fn && r->fn->createNil) - obj = r->fn->createNil(cur); - else - obj = (void*)REDIS_REPLY_NIL; - success = 1; - } else { - /* Only continue when the buffer contains the entire bulk item. */ - bytelen += len+2; /* include \r\n */ - if (r->pos+bytelen <= r->len) { - if (r->fn && r->fn->createString) - obj = r->fn->createString(cur,s+2,len); - else - obj = (void*)REDIS_REPLY_STRING; - success = 1; - } - } - - /* Proceed when obj was created. */ - if (success) { - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - r->pos += bytelen; - - /* Set reply if this is the root object. */ - if (r->ridx == 0) r->reply = obj; - moveToNextTask(r); - return REDIS_OK; - } - } - - return REDIS_ERR; -} - -static int processMultiBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - void *obj; - char *p; - long elements; - int root = 0; - - /* Set error for nested multi bulks with depth > 7 */ - if (r->ridx == 8) { - __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "No support for nested multi bulk replies with depth > 7"); - return REDIS_ERR; - } - - if ((p = readLine(r,NULL)) != NULL) { - elements = readLongLong(p); - root = (r->ridx == 0); - - if (elements == -1) { - if (r->fn && r->fn->createNil) - obj = r->fn->createNil(cur); - else - obj = (void*)REDIS_REPLY_NIL; - - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - moveToNextTask(r); - } else { - if (r->fn && r->fn->createArray) - obj = r->fn->createArray(cur,elements); - else - obj = (void*)REDIS_REPLY_ARRAY; - - if (obj == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - /* Modify task stack when there are more than 0 elements. */ - if (elements > 0) { - cur->elements = elements; - cur->obj = obj; - r->ridx++; - r->rstack[r->ridx].type = -1; - r->rstack[r->ridx].elements = -1; - r->rstack[r->ridx].idx = 0; - r->rstack[r->ridx].obj = NULL; - r->rstack[r->ridx].parent = cur; - r->rstack[r->ridx].privdata = r->privdata; - } else { - moveToNextTask(r); - } - } - - /* Set reply if this is the root object. */ - if (root) r->reply = obj; - return REDIS_OK; - } - - return REDIS_ERR; -} - -static int processItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); - char *p; - - /* check if we need to read type */ - if (cur->type < 0) { - if ((p = readBytes(r,1)) != NULL) { - switch (p[0]) { - case '-': - cur->type = REDIS_REPLY_ERROR; - break; - case '+': - cur->type = REDIS_REPLY_STATUS; - break; - case ':': - cur->type = REDIS_REPLY_INTEGER; - break; - case '$': - cur->type = REDIS_REPLY_STRING; - break; - case '*': - cur->type = REDIS_REPLY_ARRAY; - break; - default: - __redisReaderSetErrorProtocolByte(r,*p); - return REDIS_ERR; - } - } else { - /* could not consume 1 byte */ - return REDIS_ERR; - } - } - - /* process typed item */ - switch(cur->type) { - case REDIS_REPLY_ERROR: - case REDIS_REPLY_STATUS: - case REDIS_REPLY_INTEGER: - return processLineItem(r); - case REDIS_REPLY_STRING: - return processBulkItem(r); - case REDIS_REPLY_ARRAY: - return processMultiBulkItem(r); - default: - assert(NULL); - return REDIS_ERR; /* Avoid warning. */ - } -} - -redisReader *redisReaderCreate(void) { - redisReader *r; - - r = calloc(sizeof(redisReader),1); - if (r == NULL) - return NULL; - - r->err = 0; - r->errstr[0] = '\0'; - r->fn = &defaultFunctions; - r->buf = sdsempty(); - r->maxbuf = REDIS_READER_MAX_BUF; - if (r->buf == NULL) { - free(r); - return NULL; - } - - r->ridx = -1; - return r; -} - -void redisReaderFree(redisReader *r) { - if (r->reply != NULL && r->fn && r->fn->freeObject) - r->fn->freeObject(r->reply); - if (r->buf != NULL) - sdsfree(r->buf); - free(r); -} - -int redisReaderFeed(redisReader *r, const char *buf, size_t len) { - sds newbuf; - - /* Return early when this reader is in an erroneous state. */ - if (r->err) - return REDIS_ERR; - - /* Copy the provided buffer. */ - if (buf != NULL && len >= 1) { - /* Destroy internal buffer when it is empty and is quite large. */ - if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { - sdsfree(r->buf); - r->buf = sdsempty(); - r->pos = 0; - - /* r->buf should not be NULL since we just free'd a larger one. */ - assert(r->buf != NULL); - } - - newbuf = sdscatlen(r->buf,buf,len); - if (newbuf == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } - - r->buf = newbuf; - r->len = sdslen(r->buf); - } - - return REDIS_OK; -} - -int redisReaderGetReply(redisReader *r, void **reply) { - /* Default target pointer to NULL. */ - if (reply != NULL) - *reply = NULL; - - /* Return early when this reader is in an erroneous state. */ - if (r->err) - return REDIS_ERR; - - /* When the buffer is empty, there will never be a reply. */ - if (r->len == 0) - return REDIS_OK; - - /* Set first item to process when the stack is empty. */ - if (r->ridx == -1) { - r->rstack[0].type = -1; - r->rstack[0].elements = -1; - r->rstack[0].idx = -1; - r->rstack[0].obj = NULL; - r->rstack[0].parent = NULL; - r->rstack[0].privdata = r->privdata; - r->ridx = 0; - } - - /* Process items in reply. */ - while (r->ridx >= 0) - if (processItem(r) != REDIS_OK) - break; - - /* Return ASAP when an error occurred. */ - if (r->err) - return REDIS_ERR; - - /* Discard part of the buffer when we've consumed at least 1k, to avoid - * doing unnecessary calls to memmove() in sds.c. */ - if (r->pos >= 1024) { - r->buf = sdsrange(r->buf,r->pos,-1); - r->pos = 0; - r->len = sdslen(r->buf); - } - - /* Emit a reply when there is one. */ - if (r->ridx == -1) { - if (reply != NULL) - *reply = r->reply; - r->reply = NULL; - } - return REDIS_OK; -} - -/* Calculate the number of bytes needed to represent an integer as string. */ -static int intlen(int i) { - int len = 0; - if (i < 0) { - len++; - i = -i; - } - do { - len++; - i /= 10; - } while(i); - return len; +/* Return the number of digits of 'v' when converted to string in radix 10. + * Implementation borrowed from link in redis/src/util.c:string2ll(). */ +static uint32_t countDigits(uint64_t v) { + uint32_t result = 1; + for (;;) { + if (v < 10) return result; + if (v < 100) return result + 1; + if (v < 1000) return result + 2; + if (v < 10000) return result + 3; + v /= 10000U; + result += 4; + } } /* Helper that calculates the bulk length given a certain string length. */ static size_t bulklen(size_t len) { - return 1+intlen(len)+2+len+2; + return 1+countDigits(len)+2+len+2; } int redisvFormatCommand(char **target, const char *format, va_list ap) { @@ -692,6 +216,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { char **curargv = NULL, **newargv = NULL; int argc = 0; int totlen = 0; + int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */ int j; /* Abort if there is not target to set */ @@ -708,19 +233,19 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { if (*c == ' ') { if (touched) { newargv = realloc(curargv,sizeof(char*)*(argc+1)); - if (newargv == NULL) goto err; + if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; totlen += bulklen(sdslen(curarg)); /* curarg is put in argv so it can be overwritten. */ curarg = sdsempty(); - if (curarg == NULL) goto err; + if (curarg == NULL) goto memory_err; touched = 0; } } else { newarg = sdscatlen(curarg,c,1); - if (newarg == NULL) goto err; + if (newarg == NULL) goto memory_err; curarg = newarg; touched = 1; } @@ -751,17 +276,14 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { /* Try to detect printf format */ { static const char intfmts[] = "diouxX"; + static const char flags[] = "#0-+ "; char _format[16]; const char *_p = c+1; size_t _l = 0; va_list _cpy; /* Flags */ - if (*_p != '\0' && *_p == '#') _p++; - if (*_p != '\0' && *_p == '0') _p++; - if (*_p != '\0' && *_p == '-') _p++; - if (*_p != '\0' && *_p == ' ') _p++; - if (*_p != '\0' && *_p == '+') _p++; + while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++; /* Field width */ while (*_p != '\0' && isdigit(*_p)) _p++; @@ -829,7 +351,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { fmt_invalid: va_end(_cpy); - goto err; + goto format_err; fmt_valid: _l = (_p+1)-c; @@ -848,7 +370,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { } } - if (newarg == NULL) goto err; + if (newarg == NULL) goto memory_err; curarg = newarg; touched = 1; @@ -860,7 +382,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { /* Add the last argument if needed */ if (touched) { newargv = realloc(curargv,sizeof(char*)*(argc+1)); - if (newargv == NULL) goto err; + if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; totlen += bulklen(sdslen(curarg)); @@ -872,11 +394,11 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { curarg = NULL; /* Add bytes needed to hold multi bulk count */ - totlen += 1+intlen(argc)+2; + totlen += 1+countDigits(argc)+2; /* Build the command at protocol level */ cmd = malloc(totlen+1); - if (cmd == NULL) goto err; + if (cmd == NULL) goto memory_err; pos = sprintf(cmd,"*%d\r\n",argc); for (j = 0; j < argc; j++) { @@ -894,20 +416,29 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { *target = cmd; return totlen; -err: - while(argc--) - sdsfree(curargv[argc]); - free(curargv); +format_err: + error_type = -2; + goto cleanup; - if (curarg != NULL) - sdsfree(curarg); +memory_err: + error_type = -1; + goto cleanup; + +cleanup: + if (curargv) { + while(argc--) + sdsfree(curargv[argc]); + free(curargv); + } + + sdsfree(curarg); /* No need to check cmd since it is the last statement that can fail, * but do it anyway to be as defensive as possible. */ if (cmd != NULL) free(cmd); - return -1; + return error_type; } /* Format a command according to the Redis protocol. This function @@ -917,7 +448,7 @@ err: * %b represents a binary safe string * * When using %b you need to provide both the pointer to the string - * and the length in bytes. Examples: + * and the length in bytes as a size_t. Examples: * * len = redisFormatCommand(target, "GET %s", mykey); * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen); @@ -928,9 +459,69 @@ int redisFormatCommand(char **target, const char *format, ...) { va_start(ap,format); len = redisvFormatCommand(target,format,ap); va_end(ap); + + /* The API says "-1" means bad result, but we now also return "-2" in some + * cases. Force the return value to always be -1. */ + if (len < 0) + len = -1; + return len; } +/* Format a command according to the Redis protocol using an sds string and + * sdscatfmt for the processing of arguments. This function takes the + * number of arguments, an array with arguments and an array with their + * lengths. If the latter is set to NULL, strlen will be used to compute the + * argument lengths. + */ +int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, + const size_t *argvlen) +{ + sds cmd; + unsigned long long totlen; + int j; + size_t len; + + /* Abort on a NULL target */ + if (target == NULL) + return -1; + + /* Calculate our total size */ + totlen = 1+countDigits(argc)+2; + for (j = 0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + totlen += bulklen(len); + } + + /* Use an SDS string for command construction */ + cmd = sdsempty(); + if (cmd == NULL) + return -1; + + /* We already know how much storage we need */ + cmd = sdsMakeRoomFor(cmd, totlen); + if (cmd == NULL) + return -1; + + /* Construct command */ + cmd = sdscatfmt(cmd, "*%i\r\n", argc); + for (j=0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + cmd = sdscatfmt(cmd, "$%u\r\n", len); + cmd = sdscatlen(cmd, argv[j], len); + cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); + } + + assert(sdslen(cmd)==totlen); + + *target = cmd; + return totlen; +} + +void redisFreeSdsCommand(sds cmd) { + sdsfree(cmd); +} + /* Format a command according to the Redis protocol. This function takes the * number of arguments, an array with arguments and an array with their * lengths. If the latter is set to NULL, strlen will be used to compute the @@ -942,8 +533,12 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz size_t len; int totlen, j; + /* Abort on a NULL target */ + if (target == NULL) + return -1; + /* Calculate number of bytes needed for the command */ - totlen = 1+intlen(argc)+2; + totlen = 1+countDigits(argc)+2; for (j = 0; j < argc; j++) { len = argvlen ? argvlen[j] : strlen(argv[j]); totlen += bulklen(len); @@ -970,6 +565,10 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz return totlen; } +void redisFreeCommand(char *cmd) { + free(cmd); +} + void __redisSetError(redisContext *c, int type, const char *str) { size_t len; @@ -982,10 +581,14 @@ void __redisSetError(redisContext *c, int type, const char *str) { } else { /* Only REDIS_ERR_IO may lack a description! */ assert(type == REDIS_ERR_IO); - strerror_r(errno,c->errstr,sizeof(c->errstr)); + __redis_strerror_r(errno, c->errstr, sizeof(c->errstr)); } } +redisReader *redisReaderCreate(void) { + return redisReaderCreateWithFunctions(&defaultFunctions); +} + static redisContext *redisContextInit(void) { redisContext *c; @@ -997,71 +600,192 @@ static redisContext *redisContextInit(void) { c->errstr[0] = '\0'; c->obuf = sdsempty(); c->reader = redisReaderCreate(); + c->tcp.host = NULL; + c->tcp.source_addr = NULL; + c->unix_sock.path = NULL; + c->timeout = NULL; + + if (c->obuf == NULL || c->reader == NULL) { + redisFree(c); + return NULL; + } + return c; } void redisFree(redisContext *c) { + if (c == NULL) + return; if (c->fd > 0) close(c->fd); if (c->obuf != NULL) sdsfree(c->obuf); if (c->reader != NULL) redisReaderFree(c->reader); + if (c->tcp.host) + free(c->tcp.host); + if (c->tcp.source_addr) + free(c->tcp.source_addr); + if (c->unix_sock.path) + free(c->unix_sock.path); + if (c->timeout) + free(c->timeout); free(c); } +int redisFreeKeepFd(redisContext *c) { + int fd = c->fd; + c->fd = -1; + redisFree(c); + return fd; +} + +int redisReconnect(redisContext *c) { + c->err = 0; + memset(c->errstr, '\0', strlen(c->errstr)); + + if (c->fd > 0) { + close(c->fd); + } + + sdsfree(c->obuf); + redisReaderFree(c->reader); + + c->obuf = sdsempty(); + c->reader = redisReaderCreate(); + + if (c->connection_type == REDIS_CONN_TCP) { + return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, + c->timeout, c->tcp.source_addr); + } else if (c->connection_type == REDIS_CONN_UNIX) { + return redisContextConnectUnix(c, c->unix_sock.path, c->timeout); + } else { + /* Something bad happened here and shouldn't have. There isn't + enough information in the context to reconnect. */ + __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect"); + } + + return REDIS_ERR; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c = redisContextInit(); + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectTcp(c,ip,port,NULL); return c; } -redisContext *redisConnectWithTimeout(const char *ip, int port, struct timeval tv) { - redisContext *c = redisContextInit(); +redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectTcp(c,ip,port,&tv); return c; } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c = redisContextInit(); + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags &= ~REDIS_BLOCK; redisContextConnectTcp(c,ip,port,NULL); return c; } -redisContext *redisConnectUnix(const char *path) { +redisContext *redisConnectBindNonBlock(const char *ip, int port, + const char *source_addr) { redisContext *c = redisContextInit(); + c->flags &= ~REDIS_BLOCK; + redisContextConnectBindTcp(c,ip,port,NULL,source_addr); + return c; +} + +redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisContextInit(); + c->flags &= ~REDIS_BLOCK; + c->flags |= REDIS_REUSEADDR; + redisContextConnectBindTcp(c,ip,port,NULL,source_addr); + return c; +} + +redisContext *redisConnectUnix(const char *path) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectUnix(c,path,NULL); return c; } -redisContext *redisConnectUnixWithTimeout(const char *path, struct timeval tv) { - redisContext *c = redisContextInit(); +redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectUnix(c,path,&tv); return c; } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c = redisContextInit(); + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags &= ~REDIS_BLOCK; redisContextConnectUnix(c,path,NULL); return c; } +redisContext *redisConnectFd(int fd) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->fd = fd; + c->flags |= REDIS_BLOCK | REDIS_CONNECTED; + return c; +} + /* Set read/write timeout on a blocking socket. */ -int redisSetTimeout(redisContext *c, struct timeval tv) { +int redisSetTimeout(redisContext *c, const struct timeval tv) { if (c->flags & REDIS_BLOCK) return redisContextSetTimeout(c,tv); return REDIS_ERR; } +/* Enable connection KeepAlive. */ +int redisEnableKeepAlive(redisContext *c) { + if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK) + return REDIS_ERR; + return REDIS_OK; +} + /* Use this function to handle a read event on the descriptor. It will try * and read some bytes from the socket and feed them to the reply parser. * @@ -1077,7 +801,7 @@ int redisBufferRead(redisContext *c) { nread = read(c->fd,buf,sizeof(buf)); if (nread == -1) { - if (errno == EAGAIN && !(c->flags & REDIS_BLOCK)) { + if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ } else { __redisSetError(c,REDIS_ERR_IO,NULL); @@ -1098,10 +822,10 @@ int redisBufferRead(redisContext *c) { /* Write the output buffer to the socket. * * Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was - * succesfully written to the socket. When the buffer is empty after the + * successfully written to the socket. When the buffer is empty after the * write operation, "done" is set to 1 (if given). * - * Returns REDIS_ERR if an error occured trying to write and sets + * Returns REDIS_ERR if an error occurred trying to write and sets * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { @@ -1114,7 +838,7 @@ int redisBufferWrite(redisContext *c, int *done) { if (sdslen(c->obuf) > 0) { nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); if (nwritten == -1) { - if (errno == EAGAIN && !(c->flags & REDIS_BLOCK)) { + if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ } else { __redisSetError(c,REDIS_ERR_IO,NULL); @@ -1125,7 +849,7 @@ int redisBufferWrite(redisContext *c, int *done) { sdsfree(c->obuf); c->obuf = sdsempty(); } else { - c->obuf = sdsrange(c->obuf,nwritten,-1); + sdsrange(c->obuf,nwritten,-1); } } } @@ -1180,7 +904,7 @@ int redisGetReply(redisContext *c, void **reply) { * is used, you need to call redisGetReply yourself to retrieve * the reply (or replies in pub/sub). */ -int __redisAppendCommand(redisContext *c, char *cmd, size_t len) { +int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) { sds newbuf; newbuf = sdscatlen(c->obuf,cmd,len); @@ -1193,6 +917,15 @@ int __redisAppendCommand(redisContext *c, char *cmd, size_t len) { return REDIS_OK; } +int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) { + + if (__redisAppendCommand(c, cmd, len) != REDIS_OK) { + return REDIS_ERR; + } + + return REDIS_OK; +} + int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { char *cmd; int len; @@ -1201,6 +934,9 @@ int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { if (len == -1) { __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); return REDIS_ERR; + } else if (len == -2) { + __redisSetError(c,REDIS_ERR_OTHER,"Invalid format string"); + return REDIS_ERR; } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { @@ -1223,21 +959,21 @@ int redisAppendCommand(redisContext *c, const char *format, ...) { } int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { - char *cmd; + sds cmd; int len; - len = redisFormatCommandArgv(&cmd,argc,argv,argvlen); + len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); if (len == -1) { __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); return REDIS_ERR; } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - free(cmd); + sdsfree(cmd); return REDIS_ERR; } - free(cmd); + sdsfree(cmd); return REDIS_OK; } @@ -1248,7 +984,7 @@ int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const s * context is non-blocking, the "reply" pointer will not be used and the * command is simply appended to the write buffer. * - * Returns the reply when a reply was succesfully retrieved. Returns NULL + * Returns the reply when a reply was successfully retrieved. Returns NULL * otherwise. When NULL is returned in a blocking context, the error field * in the context will be set. */ diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index b922831e3..423d5e504 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -1,6 +1,8 @@ /* * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger * * All rights reserved. * @@ -31,26 +33,16 @@ #ifndef __HIREDIS_H #define __HIREDIS_H -#include /* for size_t */ +#include "read.h" #include /* for va_list */ #include /* for struct timeval */ +#include /* uintXX_t, etc */ +#include "sds.h" /* for sds */ #define HIREDIS_MAJOR 0 -#define HIREDIS_MINOR 10 -#define HIREDIS_PATCH 1 - -#define REDIS_ERR -1 -#define REDIS_OK 0 - -/* When an error occurs, the err flag in a context is set to hold the type of - * error that occured. REDIS_ERR_IO means there was an I/O error and you - * should use the "errno" variable to find out what is wrong. - * For other values, the "errstr" field will hold a description. */ -#define REDIS_ERR_IO 1 /* Error in read or write */ -#define REDIS_ERR_EOF 3 /* End of file */ -#define REDIS_ERR_PROTOCOL 4 /* Protocol error */ -#define REDIS_ERR_OOM 5 /* Out of memory */ -#define REDIS_ERR_OTHER 2 /* Everything else... */ +#define HIREDIS_MINOR 13 +#define HIREDIS_PATCH 3 +#define HIREDIS_SONAME 0.13 /* Connection type can be blocking or non-blocking and is set in the * least significant bit of the flags field in redisContext. */ @@ -79,14 +71,38 @@ /* Flag that is set when monitor mode is active */ #define REDIS_MONITORING 0x40 -#define REDIS_REPLY_STRING 1 -#define REDIS_REPLY_ARRAY 2 -#define REDIS_REPLY_INTEGER 3 -#define REDIS_REPLY_NIL 4 -#define REDIS_REPLY_STATUS 5 -#define REDIS_REPLY_ERROR 6 +/* Flag that is set when we should set SO_REUSEADDR before calling bind() */ +#define REDIS_REUSEADDR 0x80 -#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ +#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ + +/* number of times we retry to connect in the case of EADDRNOTAVAIL and + * SO_REUSEADDR is being used. */ +#define REDIS_CONNECT_RETRIES 10 + +/* strerror_r has two completely different prototypes and behaviors + * depending on system issues, so we need to operate on the error buffer + * differently depending on which strerror_r we're using. */ +#ifndef _GNU_SOURCE +/* "regular" POSIX strerror_r that does the right thing. */ +#define __redis_strerror_r(errno, buf, len) \ + do { \ + strerror_r((errno), (buf), (len)); \ + } while (0) +#else +/* "bad" GNU strerror_r we need to clean up after. */ +#define __redis_strerror_r(errno, buf, len) \ + do { \ + char *err_str = strerror_r((errno), (buf), (len)); \ + /* If return value _isn't_ the start of the buffer we passed in, \ + * then GNU strerror_r returned an internal static buffer and we \ + * need to copy the result into our private buffer. */ \ + if (err_str != (buf)) { \ + strncpy((buf), err_str, ((len) - 1)); \ + buf[(len)-1] = '\0'; \ + } \ + } while (0) +#endif #ifdef __cplusplus extern "C" { @@ -96,61 +112,13 @@ extern "C" { typedef struct redisReply { int type; /* REDIS_REPLY_* */ long long integer; /* The integer when type is REDIS_REPLY_INTEGER */ - int len; /* Length of string */ + size_t len; /* Length of string */ char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ } redisReply; -typedef struct redisReadTask { - int type; - int elements; /* number of elements in multibulk container */ - int idx; /* index in parent (array) object */ - void *obj; /* holds user-generated value for a read task */ - struct redisReadTask *parent; /* parent task */ - void *privdata; /* user-settable arbitrary field */ -} redisReadTask; - -typedef struct redisReplyObjectFunctions { - void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); - void *(*createInteger)(const redisReadTask*, long long); - void *(*createNil)(const redisReadTask*); - void (*freeObject)(void*); -} redisReplyObjectFunctions; - -/* State for the protocol parser */ -typedef struct redisReader { - int err; /* Error flags, 0 when there is no error */ - char errstr[128]; /* String representation of error when applicable */ - - char *buf; /* Read buffer */ - size_t pos; /* Buffer cursor */ - size_t len; /* Buffer length */ - size_t maxbuf; /* Max length of unused buffer */ - - redisReadTask rstack[9]; - int ridx; /* Index of current read task */ - void *reply; /* Temporary reply pointer */ - - redisReplyObjectFunctions *fn; - void *privdata; -} redisReader; - -/* Public API for the protocol parser. */ redisReader *redisReaderCreate(void); -void redisReaderFree(redisReader *r); -int redisReaderFeed(redisReader *r, const char *buf, size_t len); -int redisReaderGetReply(redisReader *r, void **reply); - -/* Backwards compatibility, can be removed on big version bump. */ -#define redisReplyReaderCreate redisReaderCreate -#define redisReplyReaderFree redisReaderFree -#define redisReplyReaderFeed redisReaderFeed -#define redisReplyReaderGetReply redisReaderGetReply -#define redisReplyReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p)) -#define redisReplyReaderGetObject(_r) (((redisReader*)(_r))->reply) -#define redisReplyReaderGetError(_r) (((redisReader*)(_r))->errstr) /* Function to free the reply objects hiredis returns by default. */ void freeReplyObject(void *reply); @@ -159,6 +127,14 @@ void freeReplyObject(void *reply); int redisvFormatCommand(char **target, const char *format, va_list ap); int redisFormatCommand(char **target, const char *format, ...); int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); +int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen); +void redisFreeCommand(char *cmd); +void redisFreeSdsCommand(sds cmd); + +enum redisConnectionType { + REDIS_CONN_TCP, + REDIS_CONN_UNIX +}; /* Context for a connection to Redis */ typedef struct redisContext { @@ -168,16 +144,49 @@ typedef struct redisContext { int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ + + enum redisConnectionType connection_type; + struct timeval *timeout; + + struct { + char *host; + char *source_addr; + int port; + } tcp; + + struct { + char *path; + } unix_sock; + } redisContext; redisContext *redisConnect(const char *ip, int port); -redisContext *redisConnectWithTimeout(const char *ip, int port, struct timeval tv); +redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); +redisContext *redisConnectBindNonBlock(const char *ip, int port, + const char *source_addr); +redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, + const char *source_addr); redisContext *redisConnectUnix(const char *path); -redisContext *redisConnectUnixWithTimeout(const char *path, struct timeval tv); +redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -int redisSetTimeout(redisContext *c, struct timeval tv); +redisContext *redisConnectFd(int fd); + +/** + * Reconnect the given context using the saved information. + * + * This re-uses the exact same connect options as in the initial connection. + * host, ip (or path), timeout and bind address are reused, + * flags are used unmodified from the existing context. + * + * Returns REDIS_OK on successful connect or REDIS_ERR otherwise. + */ +int redisReconnect(redisContext *c); + +int redisSetTimeout(redisContext *c, const struct timeval tv); +int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); +int redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); @@ -188,6 +197,10 @@ int redisBufferWrite(redisContext *c, int *done); int redisGetReply(redisContext *c, void **reply); int redisGetReplyFromReader(redisContext *c, void **reply); +/* Write a formatted command to the output buffer. Use these functions in blocking mode + * to get a pipeline of commands. */ +int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len); + /* Write a command to the output buffer. Use these functions in blocking mode * to get a pipeline of commands. */ int redisvAppendCommand(redisContext *c, const char *format, va_list ap); diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index 82ab2b468..7d4120985 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -1,7 +1,9 @@ /* Extracted from anet.c to work properly with Hiredis error reporting. * - * Copyright (c) 2006-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger * * All rights reserved. * @@ -47,6 +49,7 @@ #include #include #include +#include #include "net.h" #include "sds.h" @@ -54,21 +57,28 @@ /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); +static void redisContextCloseFd(redisContext *c) { + if (c && c->fd >= 0) { + close(c->fd); + c->fd = -1; + } +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { - char buf[128]; + char buf[128] = { 0 }; size_t len = 0; if (prefix != NULL) len = snprintf(buf,sizeof(buf),"%s: ",prefix); - strerror_r(errno,buf+len,sizeof(buf)-len); + __redis_strerror_r(errno, (char *)(buf + len), sizeof(buf) - len); __redisSetError(c,type,buf); } -static int redisSetReuseAddr(redisContext *c, int fd) { +static int redisSetReuseAddr(redisContext *c) { int on = 1; - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { + if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } return REDIS_OK; @@ -80,23 +90,24 @@ static int redisCreateSocket(redisContext *c, int type) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } + c->fd = s; if (type == AF_INET) { - if (redisSetReuseAddr(c,s) == REDIS_ERR) { + if (redisSetReuseAddr(c) == REDIS_ERR) { return REDIS_ERR; } } - return s; + return REDIS_OK; } -static int redisSetBlocking(redisContext *c, int fd, int blocking) { +static int redisSetBlocking(redisContext *c, int blocking) { int flags; /* Set the socket nonblocking. * Note that fcntl(2) for F_GETFL and F_SETFL can't be * interrupted by a signal. */ - if ((flags = fcntl(fd, F_GETFL)) == -1) { + if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } @@ -105,19 +116,61 @@ static int redisSetBlocking(redisContext *c, int fd, int blocking) { else flags |= O_NONBLOCK; - if (fcntl(fd, F_SETFL, flags) == -1) { + if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } return REDIS_OK; } -static int redisSetTcpNoDelay(redisContext *c, int fd) { +int redisKeepAlive(redisContext *c, int interval) { + int val = 1; + int fd = c->fd; + + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = interval; + +#ifdef _OSX + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } +#else +#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__) + val = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = interval/3; + if (val == 0) val = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = 3; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } +#endif +#endif + + return REDIS_OK; +} + +static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; - if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { + if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } return REDIS_OK; @@ -125,18 +178,15 @@ static int redisSetTcpNoDelay(redisContext *c, int fd) { #define __MAX_MSEC (((LONG_MAX) - 999) / 1000) -static int redisContextWaitReady(redisContext *c, int fd, const struct timeval *timeout) { - struct pollfd wfd[1]; - long msec; - - msec = -1; - wfd[0].fd = fd; - wfd[0].events = POLLOUT; +static int redisContextTimeoutMsec(redisContext *c, long *result) +{ + const struct timeval *timeout = c->timeout; + long msec = -1; /* Only use timeout when not NULL. */ if (timeout != NULL) { if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) { - close(fd); + *result = msec; return REDIS_ERR; } @@ -147,52 +197,60 @@ static int redisContextWaitReady(redisContext *c, int fd, const struct timeval * } } + *result = msec; + return REDIS_OK; +} + +static int redisContextWaitReady(redisContext *c, long msec) { + struct pollfd wfd[1]; + + wfd[0].fd = c->fd; + wfd[0].events = POLLOUT; + if (errno == EINPROGRESS) { int res; if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } - if (redisCheckSocketError(c, fd) != REDIS_OK) + if (redisCheckSocketError(c) != REDIS_OK) return REDIS_ERR; return REDIS_OK; } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } -int redisCheckSocketError(redisContext *c, int fd) { +int redisCheckSocketError(redisContext *c) { int err = 0; socklen_t errlen = sizeof(err); - if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { + if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)"); - close(fd); return REDIS_ERR; } if (err) { errno = err; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); return REDIS_ERR; } return REDIS_OK; } -int redisContextSetTimeout(redisContext *c, struct timeval tv) { +int redisContextSetTimeout(redisContext *c, const struct timeval tv) { if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; @@ -204,44 +262,141 @@ int redisContextSetTimeout(redisContext *c, struct timeval tv) { return REDIS_OK; } -int redisContextConnectTcp(redisContext *c, const char *addr, int port, struct timeval *timeout) { - int s, rv; +static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr) { + int s, rv, n; char _port[6]; /* strlen("65535"); */ - struct addrinfo hints, *servinfo, *p; + struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); + int reuseaddr = (c->flags & REDIS_REUSEADDR); + int reuses = 0; + long timeout_msec = -1; + + servinfo = NULL; + c->connection_type = REDIS_CONN_TCP; + c->tcp.port = port; + + /* We need to take possession of the passed parameters + * to make them reusable for a reconnect. + * We also carefully check we don't free data we already own, + * as in the case of the reconnect method. + * + * This is a bit ugly, but atleast it works and doesn't leak memory. + **/ + if (c->tcp.host != addr) { + if (c->tcp.host) + free(c->tcp.host); + + c->tcp.host = strdup(addr); + } + + if (timeout) { + if (c->timeout != timeout) { + if (c->timeout == NULL) + c->timeout = malloc(sizeof(struct timeval)); + + memcpy(c->timeout, timeout, sizeof(struct timeval)); + } + } else { + if (c->timeout) + free(c->timeout); + c->timeout = NULL; + } + + if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) { + __redisSetError(c, REDIS_ERR_IO, "Invalid timeout specified"); + goto error; + } + + if (source_addr == NULL) { + free(c->tcp.source_addr); + c->tcp.source_addr = NULL; + } else if (c->tcp.source_addr != source_addr) { + free(c->tcp.source_addr); + c->tcp.source_addr = strdup(source_addr); + } snprintf(_port, 6, "%d", port); memset(&hints,0,sizeof(hints)); hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; - if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { - __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); - return REDIS_ERR; + /* Try with IPv6 if no IPv4 address was found. We do it in this order since + * in a Redis client you can't afford to test if you have IPv6 connectivity + * as this would add latency to every connect. Otherwise a more sensible + * route could be: Use IPv6 if both addresses are available and there is IPv6 + * connectivity. */ + if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) { + hints.ai_family = AF_INET6; + if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { + __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); + return REDIS_ERR; + } } for (p = servinfo; p != NULL; p = p->ai_next) { +addrretry: if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) continue; - if (redisSetBlocking(c,s,0) != REDIS_OK) + c->fd = s; + if (redisSetBlocking(c,0) != REDIS_OK) goto error; + if (c->tcp.source_addr) { + int bound = 0; + /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */ + if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) { + char buf[128]; + snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv)); + __redisSetError(c,REDIS_ERR_OTHER,buf); + goto error; + } + + if (reuseaddr) { + n = 1; + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n, + sizeof(n)) < 0) { + goto error; + } + } + + for (b = bservinfo; b != NULL; b = b->ai_next) { + if (bind(s,b->ai_addr,b->ai_addrlen) != -1) { + bound = 1; + break; + } + } + freeaddrinfo(bservinfo); + if (!bound) { + char buf[128]; + snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno)); + __redisSetError(c,REDIS_ERR_OTHER,buf); + goto error; + } + } if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - close(s); + redisContextCloseFd(c); continue; } else if (errno == EINPROGRESS && !blocking) { /* This is ok. */ + } else if (errno == EADDRNOTAVAIL && reuseaddr) { + if (++reuses >= REDIS_CONNECT_RETRIES) { + goto error; + } else { + redisContextCloseFd(c); + goto addrretry; + } } else { - if (redisContextWaitReady(c,s,timeout) != REDIS_OK) + if (redisContextWaitReady(c,timeout_msec) != REDIS_OK) goto error; } } - if (blocking && redisSetBlocking(c,s,1) != REDIS_OK) + if (blocking && redisSetBlocking(c,1) != REDIS_OK) goto error; - if (redisSetTcpNoDelay(c,s) != REDIS_OK) + if (redisSetTcpNoDelay(c) != REDIS_OK) goto error; - c->fd = s; c->flags |= REDIS_CONNECTED; rv = REDIS_OK; goto end; @@ -260,32 +415,62 @@ end: return rv; // Need to return REDIS_OK if alright } -int redisContextConnectUnix(redisContext *c, const char *path, struct timeval *timeout) { - int s; +int redisContextConnectTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout) { + return _redisContextConnectTcp(c, addr, port, timeout, NULL); +} + +int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr) { + return _redisContextConnectTcp(c, addr, port, timeout, source_addr); +} + +int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { int blocking = (c->flags & REDIS_BLOCK); struct sockaddr_un sa; + long timeout_msec = -1; - if ((s = redisCreateSocket(c,AF_LOCAL)) < 0) + if (redisCreateSocket(c,AF_LOCAL) < 0) return REDIS_ERR; - if (redisSetBlocking(c,s,0) != REDIS_OK) + if (redisSetBlocking(c,0) != REDIS_OK) + return REDIS_ERR; + + c->connection_type = REDIS_CONN_UNIX; + if (c->unix_sock.path != path) + c->unix_sock.path = strdup(path); + + if (timeout) { + if (c->timeout != timeout) { + if (c->timeout == NULL) + c->timeout = malloc(sizeof(struct timeval)); + + memcpy(c->timeout, timeout, sizeof(struct timeval)); + } + } else { + if (c->timeout) + free(c->timeout); + c->timeout = NULL; + } + + if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; sa.sun_family = AF_LOCAL; strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(s, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { - if (redisContextWaitReady(c,s,timeout) != REDIS_OK) + if (redisContextWaitReady(c,timeout_msec) != REDIS_OK) return REDIS_ERR; } } /* Reset socket to be blocking after connect(2). */ - if (blocking && redisSetBlocking(c,s,1) != REDIS_OK) + if (blocking && redisSetBlocking(c,1) != REDIS_OK) return REDIS_ERR; - c->fd = s; c->flags |= REDIS_CONNECTED; return REDIS_OK; } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index eb8a0a1cf..2f1a0bf85 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -1,7 +1,9 @@ /* Extracted from anet.c to work properly with Hiredis error reporting. * - * Copyright (c) 2006-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2014, Pieter Noordhuis + * Copyright (c) 2015, Matt Stancliff , + * Jan-Erik Rediger * * All rights reserved. * @@ -39,9 +41,13 @@ #define AF_LOCAL AF_UNIX #endif -int redisCheckSocketError(redisContext *c, int fd); -int redisContextSetTimeout(redisContext *c, struct timeval tv); -int redisContextConnectTcp(redisContext *c, const char *addr, int port, struct timeval *timeout); -int redisContextConnectUnix(redisContext *c, const char *path, struct timeval *timeout); +int redisCheckSocketError(redisContext *c); +int redisContextSetTimeout(redisContext *c, const struct timeval tv); +int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); +int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr); +int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout); +int redisKeepAlive(redisContext *c, int interval); #endif diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c new file mode 100644 index 000000000..50333b534 --- /dev/null +++ b/deps/hiredis/read.c @@ -0,0 +1,525 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include "fmacros.h" +#include +#include +#ifndef _MSC_VER +#include +#endif +#include +#include +#include + +#include "read.h" +#include "sds.h" + +static void __redisReaderSetError(redisReader *r, int type, const char *str) { + size_t len; + + if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + r->reply = NULL; + } + + /* Clear input buffer on errors. */ + if (r->buf != NULL) { + sdsfree(r->buf); + r->buf = NULL; + r->pos = r->len = 0; + } + + /* Reset task stack. */ + r->ridx = -1; + + /* Set error. */ + r->err = type; + len = strlen(str); + len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1); + memcpy(r->errstr,str,len); + r->errstr[len] = '\0'; +} + +static size_t chrtos(char *buf, size_t size, char byte) { + size_t len = 0; + + switch(byte) { + case '\\': + case '"': + len = snprintf(buf,size,"\"\\%c\"",byte); + break; + case '\n': len = snprintf(buf,size,"\"\\n\""); break; + case '\r': len = snprintf(buf,size,"\"\\r\""); break; + case '\t': len = snprintf(buf,size,"\"\\t\""); break; + case '\a': len = snprintf(buf,size,"\"\\a\""); break; + case '\b': len = snprintf(buf,size,"\"\\b\""); break; + default: + if (isprint(byte)) + len = snprintf(buf,size,"\"%c\"",byte); + else + len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte); + break; + } + + return len; +} + +static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) { + char cbuf[8], sbuf[128]; + + chrtos(cbuf,sizeof(cbuf),byte); + snprintf(sbuf,sizeof(sbuf), + "Protocol error, got %s as reply type byte", cbuf); + __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf); +} + +static void __redisReaderSetErrorOOM(redisReader *r) { + __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory"); +} + +static char *readBytes(redisReader *r, unsigned int bytes) { + char *p; + if (r->len-r->pos >= bytes) { + p = r->buf+r->pos; + r->pos += bytes; + return p; + } + return NULL; +} + +/* Find pointer to \r\n. */ +static char *seekNewline(char *s, size_t len) { + int pos = 0; + int _len = len-1; + + /* Position should be < len-1 because the character at "pos" should be + * followed by a \n. Note that strchr cannot be used because it doesn't + * allow to search a limited length and the buffer that is being searched + * might not have a trailing NULL character. */ + while (pos < _len) { + while(pos < _len && s[pos] != '\r') pos++; + if (pos==_len) { + /* Not found. */ + return NULL; + } else { + if (s[pos+1] == '\n') { + /* Found. */ + return s+pos; + } else { + /* Continue searching. */ + pos++; + } + } + } + return NULL; +} + +/* Read a long long value starting at *s, under the assumption that it will be + * terminated by \r\n. Ambiguously returns -1 for unexpected input. */ +static long long readLongLong(char *s) { + long long v = 0; + int dec, mult = 1; + char c; + + if (*s == '-') { + mult = -1; + s++; + } else if (*s == '+') { + mult = 1; + s++; + } + + while ((c = *(s++)) != '\r') { + dec = c - '0'; + if (dec >= 0 && dec < 10) { + v *= 10; + v += dec; + } else { + /* Should not happen... */ + return -1; + } + } + + return mult*v; +} + +static char *readLine(redisReader *r, int *_len) { + char *p, *s; + int len; + + p = r->buf+r->pos; + s = seekNewline(p,(r->len-r->pos)); + if (s != NULL) { + len = s-(r->buf+r->pos); + r->pos += len+2; /* skip \r\n */ + if (_len) *_len = len; + return p; + } + return NULL; +} + +static void moveToNextTask(redisReader *r) { + redisReadTask *cur, *prv; + while (r->ridx >= 0) { + /* Return a.s.a.p. when the stack is now empty. */ + if (r->ridx == 0) { + r->ridx--; + return; + } + + cur = &(r->rstack[r->ridx]); + prv = &(r->rstack[r->ridx-1]); + assert(prv->type == REDIS_REPLY_ARRAY); + if (cur->idx == prv->elements-1) { + r->ridx--; + } else { + /* Reset the type because the next item can be anything */ + assert(cur->idx < prv->elements); + cur->type = -1; + cur->elements = -1; + cur->idx++; + return; + } + } +} + +static int processLineItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + void *obj; + char *p; + int len; + + if ((p = readLine(r,&len)) != NULL) { + if (cur->type == REDIS_REPLY_INTEGER) { + if (r->fn && r->fn->createInteger) + obj = r->fn->createInteger(cur,readLongLong(p)); + else + obj = (void*)REDIS_REPLY_INTEGER; + } else { + /* Type will be error or status. */ + if (r->fn && r->fn->createString) + obj = r->fn->createString(cur,p,len); + else + obj = (void*)(size_t)(cur->type); + } + + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + /* Set reply if this is the root object. */ + if (r->ridx == 0) r->reply = obj; + moveToNextTask(r); + return REDIS_OK; + } + + return REDIS_ERR; +} + +static int processBulkItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + void *obj = NULL; + char *p, *s; + long len; + unsigned long bytelen; + int success = 0; + + p = r->buf+r->pos; + s = seekNewline(p,r->len-r->pos); + if (s != NULL) { + p = r->buf+r->pos; + bytelen = s-(r->buf+r->pos)+2; /* include \r\n */ + len = readLongLong(p); + + if (len < 0) { + /* The nil object can always be created. */ + if (r->fn && r->fn->createNil) + obj = r->fn->createNil(cur); + else + obj = (void*)REDIS_REPLY_NIL; + success = 1; + } else { + /* Only continue when the buffer contains the entire bulk item. */ + bytelen += len+2; /* include \r\n */ + if (r->pos+bytelen <= r->len) { + if (r->fn && r->fn->createString) + obj = r->fn->createString(cur,s+2,len); + else + obj = (void*)REDIS_REPLY_STRING; + success = 1; + } + } + + /* Proceed when obj was created. */ + if (success) { + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + r->pos += bytelen; + + /* Set reply if this is the root object. */ + if (r->ridx == 0) r->reply = obj; + moveToNextTask(r); + return REDIS_OK; + } + } + + return REDIS_ERR; +} + +static int processMultiBulkItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + void *obj; + char *p; + long elements; + int root = 0; + + /* Set error for nested multi bulks with depth > 7 */ + if (r->ridx == 8) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "No support for nested multi bulk replies with depth > 7"); + return REDIS_ERR; + } + + if ((p = readLine(r,NULL)) != NULL) { + elements = readLongLong(p); + root = (r->ridx == 0); + + if (elements == -1) { + if (r->fn && r->fn->createNil) + obj = r->fn->createNil(cur); + else + obj = (void*)REDIS_REPLY_NIL; + + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + moveToNextTask(r); + } else { + if (r->fn && r->fn->createArray) + obj = r->fn->createArray(cur,elements); + else + obj = (void*)REDIS_REPLY_ARRAY; + + if (obj == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + /* Modify task stack when there are more than 0 elements. */ + if (elements > 0) { + cur->elements = elements; + cur->obj = obj; + r->ridx++; + r->rstack[r->ridx].type = -1; + r->rstack[r->ridx].elements = -1; + r->rstack[r->ridx].idx = 0; + r->rstack[r->ridx].obj = NULL; + r->rstack[r->ridx].parent = cur; + r->rstack[r->ridx].privdata = r->privdata; + } else { + moveToNextTask(r); + } + } + + /* Set reply if this is the root object. */ + if (root) r->reply = obj; + return REDIS_OK; + } + + return REDIS_ERR; +} + +static int processItem(redisReader *r) { + redisReadTask *cur = &(r->rstack[r->ridx]); + char *p; + + /* check if we need to read type */ + if (cur->type < 0) { + if ((p = readBytes(r,1)) != NULL) { + switch (p[0]) { + case '-': + cur->type = REDIS_REPLY_ERROR; + break; + case '+': + cur->type = REDIS_REPLY_STATUS; + break; + case ':': + cur->type = REDIS_REPLY_INTEGER; + break; + case '$': + cur->type = REDIS_REPLY_STRING; + break; + case '*': + cur->type = REDIS_REPLY_ARRAY; + break; + default: + __redisReaderSetErrorProtocolByte(r,*p); + return REDIS_ERR; + } + } else { + /* could not consume 1 byte */ + return REDIS_ERR; + } + } + + /* process typed item */ + switch(cur->type) { + case REDIS_REPLY_ERROR: + case REDIS_REPLY_STATUS: + case REDIS_REPLY_INTEGER: + return processLineItem(r); + case REDIS_REPLY_STRING: + return processBulkItem(r); + case REDIS_REPLY_ARRAY: + return processMultiBulkItem(r); + default: + assert(NULL); + return REDIS_ERR; /* Avoid warning. */ + } +} + +redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { + redisReader *r; + + r = calloc(sizeof(redisReader),1); + if (r == NULL) + return NULL; + + r->err = 0; + r->errstr[0] = '\0'; + r->fn = fn; + r->buf = sdsempty(); + r->maxbuf = REDIS_READER_MAX_BUF; + if (r->buf == NULL) { + free(r); + return NULL; + } + + r->ridx = -1; + return r; +} + +void redisReaderFree(redisReader *r) { + if (r->reply != NULL && r->fn && r->fn->freeObject) + r->fn->freeObject(r->reply); + if (r->buf != NULL) + sdsfree(r->buf); + free(r); +} + +int redisReaderFeed(redisReader *r, const char *buf, size_t len) { + sds newbuf; + + /* Return early when this reader is in an erroneous state. */ + if (r->err) + return REDIS_ERR; + + /* Copy the provided buffer. */ + if (buf != NULL && len >= 1) { + /* Destroy internal buffer when it is empty and is quite large. */ + if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { + sdsfree(r->buf); + r->buf = sdsempty(); + r->pos = 0; + + /* r->buf should not be NULL since we just free'd a larger one. */ + assert(r->buf != NULL); + } + + newbuf = sdscatlen(r->buf,buf,len); + if (newbuf == NULL) { + __redisReaderSetErrorOOM(r); + return REDIS_ERR; + } + + r->buf = newbuf; + r->len = sdslen(r->buf); + } + + return REDIS_OK; +} + +int redisReaderGetReply(redisReader *r, void **reply) { + /* Default target pointer to NULL. */ + if (reply != NULL) + *reply = NULL; + + /* Return early when this reader is in an erroneous state. */ + if (r->err) + return REDIS_ERR; + + /* When the buffer is empty, there will never be a reply. */ + if (r->len == 0) + return REDIS_OK; + + /* Set first item to process when the stack is empty. */ + if (r->ridx == -1) { + r->rstack[0].type = -1; + r->rstack[0].elements = -1; + r->rstack[0].idx = -1; + r->rstack[0].obj = NULL; + r->rstack[0].parent = NULL; + r->rstack[0].privdata = r->privdata; + r->ridx = 0; + } + + /* Process items in reply. */ + while (r->ridx >= 0) + if (processItem(r) != REDIS_OK) + break; + + /* Return ASAP when an error occurred. */ + if (r->err) + return REDIS_ERR; + + /* Discard part of the buffer when we've consumed at least 1k, to avoid + * doing unnecessary calls to memmove() in sds.c. */ + if (r->pos >= 1024) { + sdsrange(r->buf,r->pos,-1); + r->pos = 0; + r->len = sdslen(r->buf); + } + + /* Emit a reply when there is one. */ + if (r->ridx == -1) { + if (reply != NULL) + *reply = r->reply; + r->reply = NULL; + } + return REDIS_OK; +} diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h new file mode 100644 index 000000000..2988aa453 --- /dev/null +++ b/deps/hiredis/read.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef __HIREDIS_READ_H +#define __HIREDIS_READ_H +#include /* for size_t */ + +#define REDIS_ERR -1 +#define REDIS_OK 0 + +/* When an error occurs, the err flag in a context is set to hold the type of + * error that occurred. REDIS_ERR_IO means there was an I/O error and you + * should use the "errno" variable to find out what is wrong. + * For other values, the "errstr" field will hold a description. */ +#define REDIS_ERR_IO 1 /* Error in read or write */ +#define REDIS_ERR_EOF 3 /* End of file */ +#define REDIS_ERR_PROTOCOL 4 /* Protocol error */ +#define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_OTHER 2 /* Everything else... */ + +#define REDIS_REPLY_STRING 1 +#define REDIS_REPLY_ARRAY 2 +#define REDIS_REPLY_INTEGER 3 +#define REDIS_REPLY_NIL 4 +#define REDIS_REPLY_STATUS 5 +#define REDIS_REPLY_ERROR 6 + +#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct redisReadTask { + int type; + int elements; /* number of elements in multibulk container */ + int idx; /* index in parent (array) object */ + void *obj; /* holds user-generated value for a read task */ + struct redisReadTask *parent; /* parent task */ + void *privdata; /* user-settable arbitrary field */ +} redisReadTask; + +typedef struct redisReplyObjectFunctions { + void *(*createString)(const redisReadTask*, char*, size_t); + void *(*createArray)(const redisReadTask*, int); + void *(*createInteger)(const redisReadTask*, long long); + void *(*createNil)(const redisReadTask*); + void (*freeObject)(void*); +} redisReplyObjectFunctions; + +typedef struct redisReader { + int err; /* Error flags, 0 when there is no error */ + char errstr[128]; /* String representation of error when applicable */ + + char *buf; /* Read buffer */ + size_t pos; /* Buffer cursor */ + size_t len; /* Buffer length */ + size_t maxbuf; /* Max length of unused buffer */ + + redisReadTask rstack[9]; + int ridx; /* Index of current read task */ + void *reply; /* Temporary reply pointer */ + + redisReplyObjectFunctions *fn; + void *privdata; +} redisReader; + +/* Public API for the protocol parser. */ +redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn); +void redisReaderFree(redisReader *r); +int redisReaderFeed(redisReader *r, const char *buf, size_t len); +int redisReaderGetReply(redisReader *r, void **reply); + +#define redisReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p)) +#define redisReaderGetObject(_r) (((redisReader*)(_r))->reply) +#define redisReaderGetError(_r) (((redisReader*)(_r))->errstr) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 0af9c6720..923ffd82f 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1,6 +1,8 @@ -/* SDSLib, A C dynamic strings library +/* SDSLib 2.0 -- A C dynamic strings library * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2015, Salvatore Sanfilippo + * Copyright (c) 2015, Oran Agra + * Copyright (c) 2015, Redis Labs, Inc * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,164 +34,523 @@ #include #include #include +#include #include "sds.h" +#include "sdsalloc.h" -#ifdef SDS_ABORT_ON_OOM -static void sdsOomAbort(void) { - fprintf(stderr,"SDS: Out Of Memory (SDS_ABORT_ON_OOM defined)\n"); - abort(); -} -#endif - -sds sdsnewlen(const void *init, size_t initlen) { - struct sdshdr *sh; - - sh = malloc(sizeof(struct sdshdr)+initlen+1); -#ifdef SDS_ABORT_ON_OOM - if (sh == NULL) sdsOomAbort(); -#else - if (sh == NULL) return NULL; -#endif - sh->len = initlen; - sh->free = 0; - if (initlen) { - if (init) memcpy(sh->buf, init, initlen); - else memset(sh->buf,0,initlen); +static inline int sdsHdrSize(char type) { + switch(type&SDS_TYPE_MASK) { + case SDS_TYPE_5: + return sizeof(struct sdshdr5); + case SDS_TYPE_8: + return sizeof(struct sdshdr8); + case SDS_TYPE_16: + return sizeof(struct sdshdr16); + case SDS_TYPE_32: + return sizeof(struct sdshdr32); + case SDS_TYPE_64: + return sizeof(struct sdshdr64); } - sh->buf[initlen] = '\0'; - return (char*)sh->buf; + return 0; } +static inline char sdsReqType(size_t string_size) { + if (string_size < 32) + return SDS_TYPE_5; + if (string_size < 0xff) + return SDS_TYPE_8; + if (string_size < 0xffff) + return SDS_TYPE_16; + if (string_size < 0xffffffff) + return SDS_TYPE_32; + return SDS_TYPE_64; +} + +/* Create a new sds string with the content specified by the 'init' pointer + * and 'initlen'. + * If NULL is used for 'init' the string is initialized with zero bytes. + * + * The string is always null-termined (all the sds strings are, always) so + * even if you create an sds string with: + * + * mystring = sdsnewlen("abc",3); + * + * You can print the string with printf() as there is an implicit \0 at the + * end of the string. However the string is binary safe and can contain + * \0 characters in the middle, as the length is stored in the sds header. */ +sds sdsnewlen(const void *init, size_t initlen) { + void *sh; + sds s; + char type = sdsReqType(initlen); + /* Empty strings are usually created in order to append. Use type 8 + * since type 5 is not good at this. */ + if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8; + int hdrlen = sdsHdrSize(type); + unsigned char *fp; /* flags pointer. */ + + sh = s_malloc(hdrlen+initlen+1); + if (sh == NULL) return NULL; + if (!init) + memset(sh, 0, hdrlen+initlen+1); + s = (char*)sh+hdrlen; + fp = ((unsigned char*)s)-1; + switch(type) { + case SDS_TYPE_5: { + *fp = type | (initlen << SDS_TYPE_BITS); + break; + } + case SDS_TYPE_8: { + SDS_HDR_VAR(8,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + case SDS_TYPE_16: { + SDS_HDR_VAR(16,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + case SDS_TYPE_32: { + SDS_HDR_VAR(32,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + case SDS_TYPE_64: { + SDS_HDR_VAR(64,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + } + if (initlen && init) + memcpy(s, init, initlen); + s[initlen] = '\0'; + return s; +} + +/* Create an empty (zero length) sds string. Even in this case the string + * always has an implicit null term. */ sds sdsempty(void) { return sdsnewlen("",0); } +/* Create a new sds string starting from a null terminated C string. */ sds sdsnew(const char *init) { size_t initlen = (init == NULL) ? 0 : strlen(init); return sdsnewlen(init, initlen); } +/* Duplicate an sds string. */ sds sdsdup(const sds s) { return sdsnewlen(s, sdslen(s)); } +/* Free an sds string. No operation is performed if 's' is NULL. */ void sdsfree(sds s) { if (s == NULL) return; - free(s-sizeof(struct sdshdr)); + s_free((char*)s-sdsHdrSize(s[-1])); } +/* Set the sds string length to the length as obtained with strlen(), so + * considering as content only up to the first null term character. + * + * This function is useful when the sds string is hacked manually in some + * way, like in the following example: + * + * s = sdsnew("foobar"); + * s[2] = '\0'; + * sdsupdatelen(s); + * printf("%d\n", sdslen(s)); + * + * The output will be "2", but if we comment out the call to sdsupdatelen() + * the output will be "6" as the string was modified but the logical length + * remains 6 bytes. */ void sdsupdatelen(sds s) { - struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); int reallen = strlen(s); - sh->free += (sh->len-reallen); - sh->len = reallen; + sdssetlen(s, reallen); } -static sds sdsMakeRoomFor(sds s, size_t addlen) { - struct sdshdr *sh, *newsh; - size_t free = sdsavail(s); +/* Modify an sds string in-place to make it empty (zero length). + * However all the existing buffer is not discarded but set as free space + * so that next append operations will not require allocations up to the + * number of bytes previously available. */ +void sdsclear(sds s) { + sdssetlen(s, 0); + s[0] = '\0'; +} + +/* Enlarge the free space at the end of the sds string so that the caller + * is sure that after calling this function can overwrite up to addlen + * bytes after the end of the string, plus one more byte for nul term. + * + * Note: this does not change the *length* of the sds string as returned + * by sdslen(), but only the free buffer space we have. */ +sds sdsMakeRoomFor(sds s, size_t addlen) { + void *sh, *newsh; + size_t avail = sdsavail(s); size_t len, newlen; + char type, oldtype = s[-1] & SDS_TYPE_MASK; + int hdrlen; + + /* Return ASAP if there is enough space left. */ + if (avail >= addlen) return s; - if (free >= addlen) return s; len = sdslen(s); - sh = (void*) (s-(sizeof(struct sdshdr))); - newlen = (len+addlen)*2; - newsh = realloc(sh, sizeof(struct sdshdr)+newlen+1); -#ifdef SDS_ABORT_ON_OOM - if (newsh == NULL) sdsOomAbort(); -#else - if (newsh == NULL) return NULL; -#endif + sh = (char*)s-sdsHdrSize(oldtype); + newlen = (len+addlen); + if (newlen < SDS_MAX_PREALLOC) + newlen *= 2; + else + newlen += SDS_MAX_PREALLOC; - newsh->free = newlen - len; - return newsh->buf; + type = sdsReqType(newlen); + + /* Don't use type 5: the user is appending to the string and type 5 is + * not able to remember empty space, so sdsMakeRoomFor() must be called + * at every appending operation. */ + if (type == SDS_TYPE_5) type = SDS_TYPE_8; + + hdrlen = sdsHdrSize(type); + if (oldtype==type) { + newsh = s_realloc(sh, hdrlen+newlen+1); + if (newsh == NULL) return NULL; + s = (char*)newsh+hdrlen; + } else { + /* Since the header size changes, need to move the string forward, + * and can't use realloc */ + newsh = s_malloc(hdrlen+newlen+1); + if (newsh == NULL) return NULL; + memcpy((char*)newsh+hdrlen, s, len+1); + s_free(sh); + s = (char*)newsh+hdrlen; + s[-1] = type; + sdssetlen(s, len); + } + sdssetalloc(s, newlen); + return s; +} + +/* Reallocate the sds string so that it has no free space at the end. The + * contained string remains not altered, but next concatenation operations + * will require a reallocation. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdsRemoveFreeSpace(sds s) { + void *sh, *newsh; + char type, oldtype = s[-1] & SDS_TYPE_MASK; + int hdrlen; + size_t len = sdslen(s); + sh = (char*)s-sdsHdrSize(oldtype); + + type = sdsReqType(len); + hdrlen = sdsHdrSize(type); + if (oldtype==type) { + newsh = s_realloc(sh, hdrlen+len+1); + if (newsh == NULL) return NULL; + s = (char*)newsh+hdrlen; + } else { + newsh = s_malloc(hdrlen+len+1); + if (newsh == NULL) return NULL; + memcpy((char*)newsh+hdrlen, s, len+1); + s_free(sh); + s = (char*)newsh+hdrlen; + s[-1] = type; + sdssetlen(s, len); + } + sdssetalloc(s, len); + return s; +} + +/* Return the total size of the allocation of the specifed sds string, + * including: + * 1) The sds header before the pointer. + * 2) The string. + * 3) The free buffer at the end if any. + * 4) The implicit null term. + */ +size_t sdsAllocSize(sds s) { + size_t alloc = sdsalloc(s); + return sdsHdrSize(s[-1])+alloc+1; +} + +/* Return the pointer of the actual SDS allocation (normally SDS strings + * are referenced by the start of the string buffer). */ +void *sdsAllocPtr(sds s) { + return (void*) (s-sdsHdrSize(s[-1])); +} + +/* Increment the sds length and decrements the left free space at the + * end of the string according to 'incr'. Also set the null term + * in the new end of the string. + * + * This function is used in order to fix the string length after the + * user calls sdsMakeRoomFor(), writes something after the end of + * the current string, and finally needs to set the new length. + * + * Note: it is possible to use a negative increment in order to + * right-trim the string. + * + * Usage example: + * + * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * following schema, to cat bytes coming from the kernel to the end of an + * sds string without copying into an intermediate buffer: + * + * oldlen = sdslen(s); + * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * nread = read(fd, s+oldlen, BUFFER_SIZE); + * ... check for nread <= 0 and handle it ... + * sdsIncrLen(s, nread); + */ +void sdsIncrLen(sds s, int incr) { + unsigned char flags = s[-1]; + size_t len; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: { + unsigned char *fp = ((unsigned char*)s)-1; + unsigned char oldlen = SDS_TYPE_5_LEN(flags); + assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr))); + *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS); + len = oldlen+incr; + break; + } + case SDS_TYPE_8: { + SDS_HDR_VAR(8,s); + assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); + len = (sh->len += incr); + break; + } + case SDS_TYPE_16: { + SDS_HDR_VAR(16,s); + assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); + len = (sh->len += incr); + break; + } + case SDS_TYPE_32: { + SDS_HDR_VAR(32,s); + assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); + len = (sh->len += incr); + break; + } + case SDS_TYPE_64: { + SDS_HDR_VAR(64,s); + assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr))); + len = (sh->len += incr); + break; + } + default: len = 0; /* Just to avoid compilation warnings. */ + } + s[len] = '\0'; } /* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. */ + * the original length of the sds will be set to zero. + * + * if the specified length is smaller than the current length, no operation + * is performed. */ sds sdsgrowzero(sds s, size_t len) { - struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); - size_t totlen, curlen = sh->len; + size_t curlen = sdslen(s); if (len <= curlen) return s; s = sdsMakeRoomFor(s,len-curlen); if (s == NULL) return NULL; /* Make sure added region doesn't contain garbage */ - sh = (void*)(s-(sizeof(struct sdshdr))); memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ - totlen = sh->len+sh->free; - sh->len = len; - sh->free = totlen-sh->len; + sdssetlen(s, len); return s; } +/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the + * end of the specified sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscatlen(sds s, const void *t, size_t len) { - struct sdshdr *sh; size_t curlen = sdslen(s); s = sdsMakeRoomFor(s,len); if (s == NULL) return NULL; - sh = (void*) (s-(sizeof(struct sdshdr))); memcpy(s+curlen, t, len); - sh->len = curlen+len; - sh->free = sh->free-len; + sdssetlen(s, curlen+len); s[curlen+len] = '\0'; return s; } +/* Append the specified null termianted C string to the sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscat(sds s, const char *t) { return sdscatlen(s, t, strlen(t)); } -sds sdscpylen(sds s, char *t, size_t len) { - struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); - size_t totlen = sh->free+sh->len; +/* Append the specified sds 't' to the existing sds 's'. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatsds(sds s, const sds t) { + return sdscatlen(s, t, sdslen(t)); +} - if (totlen < len) { - s = sdsMakeRoomFor(s,len-sh->len); +/* Destructively modify the sds string 's' to hold the specified binary + * safe string pointed by 't' of length 'len' bytes. */ +sds sdscpylen(sds s, const char *t, size_t len) { + if (sdsalloc(s) < len) { + s = sdsMakeRoomFor(s,len-sdslen(s)); if (s == NULL) return NULL; - sh = (void*) (s-(sizeof(struct sdshdr))); - totlen = sh->free+sh->len; } memcpy(s, t, len); s[len] = '\0'; - sh->len = len; - sh->free = totlen-len; + sdssetlen(s, len); return s; } -sds sdscpy(sds s, char *t) { +/* Like sdscpylen() but 't' must be a null-termined string so that the length + * of the string is obtained with strlen(). */ +sds sdscpy(sds s, const char *t) { return sdscpylen(s, t, strlen(t)); } +/* Helper for sdscatlonglong() doing the actual number -> string + * conversion. 's' must point to a string with room for at least + * SDS_LLSTR_SIZE bytes. + * + * The function returns the length of the null-terminated string + * representation stored at 's'. */ +#define SDS_LLSTR_SIZE 21 +int sdsll2str(char *s, long long value) { + char *p, aux; + unsigned long long v; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + v = (value < 0) ? -value : value; + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p++ = '-'; + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Identical sdsll2str(), but for unsigned long long type. */ +int sdsull2str(char *s, unsigned long long v) { + char *p, aux; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Create an sds string from a long long value. It is much faster than: + * + * sdscatprintf(sdsempty(),"%lld\n", value); + */ +sds sdsfromlonglong(long long value) { + char buf[SDS_LLSTR_SIZE]; + int len = sdsll2str(buf,value); + + return sdsnewlen(buf,len); +} + +/* Like sdscatprintf() but gets va_list instead of being variadic. */ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { va_list cpy; - char *buf, *t; - size_t buflen = 16; + char staticbuf[1024], *buf = staticbuf, *t; + size_t buflen = strlen(fmt)*2; - while(1) { - buf = malloc(buflen); -#ifdef SDS_ABORT_ON_OOM - if (buf == NULL) sdsOomAbort(); -#else + /* We try to start using a static buffer for speed. + * If not possible we revert to heap allocation. */ + if (buflen > sizeof(staticbuf)) { + buf = s_malloc(buflen); if (buf == NULL) return NULL; -#endif + } else { + buflen = sizeof(staticbuf); + } + + /* Try with buffers two times bigger every time we fail to + * fit the string in the current buffer size. */ + while(1) { buf[buflen-2] = '\0'; va_copy(cpy,ap); vsnprintf(buf, buflen, fmt, cpy); + va_end(cpy); if (buf[buflen-2] != '\0') { - free(buf); + if (buf != staticbuf) s_free(buf); buflen *= 2; + buf = s_malloc(buflen); + if (buf == NULL) return NULL; continue; } break; } + + /* Finally concat the obtained string to the SDS string and return it. */ t = sdscat(s, buf); - free(buf); + if (buf != staticbuf) s_free(buf); return t; } +/* Append to the sds string 's' a string obtained using printf-alike format + * specifier. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("Sum is: "); + * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * + * Often you need to create a string from scratch with the printf-alike + * format. When this is the need, just use sdsempty() as the target string: + * + * s = sdscatprintf(sdsempty(), "... your format ...", args); + */ sds sdscatprintf(sds s, const char *fmt, ...) { va_list ap; char *t; @@ -199,28 +560,159 @@ sds sdscatprintf(sds s, const char *fmt, ...) { return t; } +/* This function is similar to sdscatprintf, but much faster as it does + * not rely on sprintf() family functions implemented by the libc that + * are often very slow. Moreover directly handling the sds string as + * new data is concatenated provides a performance improvement. + * + * However this function only handles an incompatible subset of printf-alike + * format specifiers: + * + * %s - C String + * %S - SDS string + * %i - signed int + * %I - 64 bit signed integer (long long, int64_t) + * %u - unsigned int + * %U - 64 bit unsigned integer (unsigned long long, uint64_t) + * %% - Verbatim "%" character. + */ +sds sdscatfmt(sds s, char const *fmt, ...) { + const char *f = fmt; + int i; + va_list ap; + + va_start(ap,fmt); + i = sdslen(s); /* Position of the next byte to write to dest str. */ + while(*f) { + char next, *str; + size_t l; + long long num; + unsigned long long unum; + + /* Make sure there is always space for at least 1 char. */ + if (sdsavail(s)==0) { + s = sdsMakeRoomFor(s,1); + } + + switch(*f) { + case '%': + next = *(f+1); + f++; + switch(next) { + case 's': + case 'S': + str = va_arg(ap,char*); + l = (next == 's') ? strlen(str) : sdslen(str); + if (sdsavail(s) < l) { + s = sdsMakeRoomFor(s,l); + } + memcpy(s+i,str,l); + sdsinclen(s,l); + i += l; + break; + case 'i': + case 'I': + if (next == 'i') + num = va_arg(ap,int); + else + num = va_arg(ap,long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsll2str(buf,num); + if (sdsavail(s) < l) { + s = sdsMakeRoomFor(s,l); + } + memcpy(s+i,buf,l); + sdsinclen(s,l); + i += l; + } + break; + case 'u': + case 'U': + if (next == 'u') + unum = va_arg(ap,unsigned int); + else + unum = va_arg(ap,unsigned long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsull2str(buf,unum); + if (sdsavail(s) < l) { + s = sdsMakeRoomFor(s,l); + } + memcpy(s+i,buf,l); + sdsinclen(s,l); + i += l; + } + break; + default: /* Handle %% and generally %. */ + s[i++] = next; + sdsinclen(s,1); + break; + } + break; + default: + s[i++] = *f; + sdsinclen(s,1); + break; + } + f++; + } + va_end(ap); + + /* Add null-term */ + s[i] = '\0'; + return s; +} + +/* Remove the part of the string from left and from right composed just of + * contiguous characters found in 'cset', that is a null terminted C string. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = sdstrim(s,"Aa. :"); + * printf("%s\n", s); + * + * Output will be just "Hello World". + */ sds sdstrim(sds s, const char *cset) { - struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); char *start, *end, *sp, *ep; size_t len; sp = start = s; ep = end = s+sdslen(s)-1; while(sp <= end && strchr(cset, *sp)) sp++; - while(ep > start && strchr(cset, *ep)) ep--; + while(ep > sp && strchr(cset, *ep)) ep--; len = (sp > ep) ? 0 : ((ep-sp)+1); - if (sh->buf != sp) memmove(sh->buf, sp, len); - sh->buf[len] = '\0'; - sh->free = sh->free+(sh->len-len); - sh->len = len; + if (s != sp) memmove(s, sp, len); + s[len] = '\0'; + sdssetlen(s,len); return s; } -sds sdsrange(sds s, int start, int end) { - struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); +/* Turn the string into a smaller (or equal) string containing only the + * substring specified by the 'start' and 'end' indexes. + * + * start and end can be negative, where -1 means the last character of the + * string, -2 the penultimate character, and so forth. + * + * The interval is inclusive, so the start and end characters will be part + * of the resulting string. + * + * The string is modified in-place. + * + * Example: + * + * s = sdsnew("Hello World"); + * sdsrange(s,1,-1); => "ello World" + */ +void sdsrange(sds s, int start, int end) { size_t newlen, len = sdslen(s); - if (len == 0) return s; + if (len == 0) return; if (start < 0) { start = len+start; if (start < 0) start = 0; @@ -240,26 +732,37 @@ sds sdsrange(sds s, int start, int end) { } else { start = 0; } - if (start && newlen) memmove(sh->buf, sh->buf+start, newlen); - sh->buf[newlen] = 0; - sh->free = sh->free+(sh->len-newlen); - sh->len = newlen; - return s; + if (start && newlen) memmove(s, s+start, newlen); + s[newlen] = 0; + sdssetlen(s,newlen); } +/* Apply tolower() to every character of the sds string 's'. */ void sdstolower(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = tolower(s[j]); } +/* Apply toupper() to every character of the sds string 's'. */ void sdstoupper(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = toupper(s[j]); } -int sdscmp(sds s1, sds s2) { +/* Compare two sds strings s1 and s2 with memcmp(). + * + * Return value: + * + * positive if s1 > s2. + * negative if s1 < s2. + * 0 if s1 and s2 are exactly the same binary string. + * + * If two strings share exactly the same prefix, but one of the two has + * additional characters, the longer string is considered to be greater than + * the smaller one. */ +int sdscmp(const sds s1, const sds s2) { size_t l1, l2, minlen; int cmp; @@ -287,14 +790,15 @@ int sdscmp(sds s1, sds s2) { * requires length arguments. sdssplit() is just the * same function but for zero-terminated strings. */ -sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { int elements = 0, slots = 5, start = 0, j; + sds *tokens; + + if (seplen < 1 || len < 0) return NULL; + + tokens = s_malloc(sizeof(sds)*slots); + if (tokens == NULL) return NULL; - sds *tokens = malloc(sizeof(sds)*slots); -#ifdef SDS_ABORT_ON_OOM - if (tokens == NULL) sdsOomAbort(); -#endif - if (seplen < 1 || len < 0 || tokens == NULL) return NULL; if (len == 0) { *count = 0; return tokens; @@ -305,26 +809,14 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { sds *newtokens; slots *= 2; - newtokens = realloc(tokens,sizeof(sds)*slots); - if (newtokens == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + newtokens = s_realloc(tokens,sizeof(sds)*slots); + if (newtokens == NULL) goto cleanup; tokens = newtokens; } /* search the separator */ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { tokens[elements] = sdsnewlen(s+start,j-start); - if (tokens[elements] == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + if (tokens[elements] == NULL) goto cleanup; elements++; start = j+seplen; j = j+seplen-1; /* skip the separator */ @@ -332,54 +824,37 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { } /* Add the final element. We are sure there is room in the tokens array. */ tokens[elements] = sdsnewlen(s+start,len-start); - if (tokens[elements] == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + if (tokens[elements] == NULL) goto cleanup; elements++; *count = elements; return tokens; -#ifndef SDS_ABORT_ON_OOM cleanup: { int i; for (i = 0; i < elements; i++) sdsfree(tokens[i]); - free(tokens); + s_free(tokens); + *count = 0; return NULL; } -#endif } +/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ void sdsfreesplitres(sds *tokens, int count) { if (!tokens) return; while(count--) sdsfree(tokens[count]); - free(tokens); + s_free(tokens); } -sds sdsfromlonglong(long long value) { - char buf[32], *p; - unsigned long long v; - - v = (value < 0) ? -value : value; - p = buf+31; /* point to the last character */ - do { - *p-- = '0'+(v%10); - v /= 10; - } while(v); - if (value < 0) *p-- = '-'; - p++; - return sdsnewlen(p,32-(p-buf)); -} - -sds sdscatrepr(sds s, char *p, size_t len) { +/* Append to the sds string "s" an escaped string representation where + * all the non-printable characters (tested with isprint()) are turned into + * escapes in the form "\n\r\a...." or "\x". + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatrepr(sds s, const char *p, size_t len) { s = sdscatlen(s,"\"",1); - if (s == NULL) return NULL; - while(len--) { switch(*p) { case '\\': @@ -399,27 +874,64 @@ sds sdscatrepr(sds s, char *p, size_t len) { break; } p++; - if (s == NULL) return NULL; } return sdscatlen(s,"\"",1); } +/* Helper function for sdssplitargs() that returns non zero if 'c' + * is a valid hex digit. */ +int is_hex_digit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); +} + +/* Helper function for sdssplitargs() that converts a hex digit into an + * integer from 0 to 15 */ +int hex_digit_to_int(char c) { + switch(c) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': case 'A': return 10; + case 'b': case 'B': return 11; + case 'c': case 'C': return 12; + case 'd': case 'D': return 13; + case 'e': case 'E': return 14; + case 'f': case 'F': return 15; + default: return 0; + } +} + /* Split a line into arguments, where every argument can be in the * following programming-language REPL-alike form: * * foo bar "newline are supported\n" and "\xff\x00otherstuff" * * The number of arguments is stored into *argc, and an array - * of sds is returned. The caller should sdsfree() all the returned - * strings and finally free() the array itself. + * of sds is returned. + * + * The caller should free the resulting array of sds strings with + * sdsfreesplitres(). * * Note that sdscatrepr() is able to convert back a string into * a quoted string in the same format sdssplitargs() is able to parse. + * + * The function returns the allocated tokens on success, even when the + * input string is empty, or NULL if the input contains unbalanced + * quotes or closed quotes followed by non space characters + * as in: "foo"bar or "foo' */ -sds *sdssplitargs(char *line, int *argc) { - char *p = line; +sds *sdssplitargs(const char *line, int *argc) { + const char *p = line; char *current = NULL; - char **vector = NULL, **_vector = NULL; + char **vector = NULL; *argc = 0; while(1) { @@ -427,17 +939,24 @@ sds *sdssplitargs(char *line, int *argc) { while(*p && isspace(*p)) p++; if (*p) { /* get a token */ - int inq=0; /* set to 1 if we are in "quotes" */ + int inq=0; /* set to 1 if we are in "quotes" */ + int insq=0; /* set to 1 if we are in 'single quotes' */ int done=0; - if (current == NULL) { - current = sdsempty(); - if (current == NULL) goto err; - } - + if (current == NULL) current = sdsempty(); while(!done) { if (inq) { - if (*p == '\\' && *(p+1)) { + if (*p == '\\' && *(p+1) == 'x' && + is_hex_digit(*(p+2)) && + is_hex_digit(*(p+3))) + { + unsigned char byte; + + byte = (hex_digit_to_int(*(p+2))*16)+ + hex_digit_to_int(*(p+3)); + current = sdscatlen(current,(char*)&byte,1); + p += 3; + } else if (*p == '\\' && *(p+1)) { char c; p++; @@ -451,7 +970,23 @@ sds *sdssplitargs(char *line, int *argc) { } current = sdscatlen(current,&c,1); } else if (*p == '"') { - /* closing quote must be followed by a space */ + /* closing quote must be followed by a space or + * nothing at all. */ + if (*(p+1) && !isspace(*(p+1))) goto err; + done=1; + } else if (!*p) { + /* unterminated quotes */ + goto err; + } else { + current = sdscatlen(current,p,1); + } + } else if (insq) { + if (*p == '\\' && *(p+1) == '\'') { + p++; + current = sdscatlen(current,"'",1); + } else if (*p == '\'') { + /* closing quote must be followed by a space or + * nothing at all. */ if (*(p+1) && !isspace(*(p+1))) goto err; done=1; } else if (!*p) { @@ -472,23 +1007,24 @@ sds *sdssplitargs(char *line, int *argc) { case '"': inq=1; break; + case '\'': + insq=1; + break; default: current = sdscatlen(current,p,1); break; } } if (*p) p++; - if (current == NULL) goto err; } /* add the token to the vector */ - _vector = realloc(vector,((*argc)+1)*sizeof(char*)); - if (_vector == NULL) goto err; - - vector = _vector; + vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); vector[*argc] = current; (*argc)++; current = NULL; } else { + /* Even on empty input string return something not NULL. */ + if (vector == NULL) vector = s_malloc(sizeof(void*)); return vector; } } @@ -496,29 +1032,76 @@ sds *sdssplitargs(char *line, int *argc) { err: while((*argc)--) sdsfree(vector[*argc]); - if (vector != NULL) free(vector); - if (current != NULL) sdsfree(current); + s_free(vector); + if (current) sdsfree(current); + *argc = 0; return NULL; } -#ifdef SDS_TEST_MAIN +/* Modify the string substituting all the occurrences of the set of + * characters specified in the 'from' string to the corresponding character + * in the 'to' array. + * + * For instance: sdsmapchars(mystring, "ho", "01", 2) + * will have the effect of turning the string "hello" into "0ell1". + * + * The function returns the sds string pointer, that is always the same + * as the input pointer since no resize is needed. */ +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = sdslen(s); + + for (j = 0; j < l; j++) { + for (i = 0; i < setlen; i++) { + if (s[j] == from[i]) { + s[j] = to[i]; + break; + } + } + } + return s; +} + +/* Join an array of C strings using the specified separator (also a C string). + * Returns the result as an sds string. */ +sds sdsjoin(char **argv, int argc, char *sep) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscat(join, argv[j]); + if (j != argc-1) join = sdscat(join,sep); + } + return join; +} + +/* Like sdsjoin, but joins an array of SDS strings. */ +sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscatsds(join, argv[j]); + if (j != argc-1) join = sdscatlen(join,sep,seplen); + } + return join; +} + +/* Wrappers to the allocators used by SDS. Note that SDS will actually + * just use the macros defined into sdsalloc.h in order to avoid to pay + * the overhead of function calls. Here we define these wrappers only for + * the programs SDS is linked to, if they want to touch the SDS internals + * even if they use a different allocator. */ +void *sds_malloc(size_t size) { return s_malloc(size); } +void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); } +void sds_free(void *ptr) { s_free(ptr); } + +#if defined(SDS_TEST_MAIN) #include +#include "testhelp.h" +#include "limits.h" -int __failed_tests = 0; -int __test_num = 0; -#define test_cond(descr,_c) do { \ - __test_num++; printf("%d - %s: ", __test_num, descr); \ - if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \ -} while(0); -#define test_report() do { \ - printf("%d tests, %d passed, %d failed\n", __test_num, \ - __test_num-__failed_tests, __failed_tests); \ - if (__failed_tests) { \ - printf("=== WARNING === We have failed tests here...\n"); \ - } \ -} while(0); - -int main(void) { +#define UNUSED(x) (void)(x) +int sdsTest(void) { { sds x = sdsnew("foo"), y; @@ -546,39 +1129,74 @@ int main(void) { sdsfree(x); x = sdscatprintf(sdsempty(),"%d",123); test_cond("sdscatprintf() seems working in the base case", - sdslen(x) == 3 && memcmp(x,"123\0",4) ==0) + sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) sdsfree(x); - x = sdstrim(sdsnew("xxciaoyyy"),"xy"); + x = sdsnew("--"); + x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); + test_cond("sdscatfmt() seems working in the base case", + sdslen(x) == 60 && + memcmp(x,"--Hello Hi! World -9223372036854775808," + "9223372036854775807--",60) == 0) + printf("[%s]\n",x); + + sdsfree(x); + x = sdsnew("--"); + x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); + test_cond("sdscatfmt() seems working with unsigned numbers", + sdslen(x) == 35 && + memcmp(x,"--4294967295,18446744073709551615--",35) == 0) + + sdsfree(x); + x = sdsnew(" x "); + sdstrim(x," x"); + test_cond("sdstrim() works when all chars match", + sdslen(x) == 0) + + sdsfree(x); + x = sdsnew(" x "); + sdstrim(x," "); + test_cond("sdstrim() works when a single char remains", + sdslen(x) == 1 && x[0] == 'x') + + sdsfree(x); + x = sdsnew("xxciaoyyy"); + sdstrim(x,"xy"); test_cond("sdstrim() correctly trims characters", sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) - y = sdsrange(sdsdup(x),1,1); + y = sdsdup(x); + sdsrange(y,1,1); test_cond("sdsrange(...,1,1)", sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),1,-1); + y = sdsdup(x); + sdsrange(y,1,-1); test_cond("sdsrange(...,1,-1)", sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),-2,-1); + y = sdsdup(x); + sdsrange(y,-2,-1); test_cond("sdsrange(...,-2,-1)", sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),2,1); + y = sdsdup(x); + sdsrange(y,2,1); test_cond("sdsrange(...,2,1)", sdslen(y) == 0 && memcmp(y,"\0",1) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),1,100); + y = sdsdup(x); + sdsrange(y,1,100); test_cond("sdsrange(...,1,100)", sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),100,100); + y = sdsdup(x); + sdsrange(y,100,100); test_cond("sdsrange(...,100,100)", sdslen(y) == 0 && memcmp(y,"\0",1) == 0) @@ -599,7 +1217,56 @@ int main(void) { x = sdsnew("aar"); y = sdsnew("bar"); test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + + sdsfree(y); + sdsfree(x); + x = sdsnewlen("\a\n\0foo\r",7); + y = sdscatrepr(sdsempty(),x,sdslen(x)); + test_cond("sdscatrepr(...data...)", + memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) + + { + unsigned int oldfree; + char *p; + int step = 10, j, i; + + sdsfree(x); + sdsfree(y); + x = sdsnew("0"); + test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0); + + /* Run the test a few times in order to hit the first two + * SDS header types. */ + for (i = 0; i < 10; i++) { + int oldlen = sdslen(x); + x = sdsMakeRoomFor(x,step); + int type = x[-1]&SDS_TYPE_MASK; + + test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen); + if (type != SDS_TYPE_5) { + test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step); + oldfree = sdsavail(x); + } + p = x+oldlen; + for (j = 0; j < step; j++) { + p[j] = 'A'+j; + } + sdsIncrLen(x,step); + } + test_cond("sdsMakeRoomFor() content", + memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0); + test_cond("sdsMakeRoomFor() final length",sdslen(x)==101); + + sdsfree(x); + } } test_report() + return 0; +} +#endif + +#ifdef SDS_TEST_MAIN +int main(void) { + return sdsTest(); } #endif diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 94f5871f5..13be75a9f 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -1,6 +1,8 @@ -/* SDSLib, A C dynamic strings library +/* SDSLib 2.0 -- A C dynamic strings library * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2015, Salvatore Sanfilippo + * Copyright (c) 2015, Oran Agra + * Copyright (c) 2015, Redis Labs, Inc * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,39 +33,198 @@ #ifndef __SDS_H #define __SDS_H +#define SDS_MAX_PREALLOC (1024*1024) + #include #include +#include typedef char *sds; -struct sdshdr { - int len; - int free; +/* Note: sdshdr5 is never used, we just access the flags byte directly. + * However is here to document the layout of type 5 SDS strings. */ +struct __attribute__ ((__packed__)) sdshdr5 { + unsigned char flags; /* 3 lsb of type, and 5 msb of string length */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr8 { + uint8_t len; /* used */ + uint8_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr16 { + uint16_t len; /* used */ + uint16_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr32 { + uint32_t len; /* used */ + uint32_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr64 { + uint64_t len; /* used */ + uint64_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ char buf[]; }; +#define SDS_TYPE_5 0 +#define SDS_TYPE_8 1 +#define SDS_TYPE_16 2 +#define SDS_TYPE_32 3 +#define SDS_TYPE_64 4 +#define SDS_TYPE_MASK 7 +#define SDS_TYPE_BITS 3 +#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))); +#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)))) +#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS) + static inline size_t sdslen(const sds s) { - struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); - return sh->len; + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + return SDS_TYPE_5_LEN(flags); + case SDS_TYPE_8: + return SDS_HDR(8,s)->len; + case SDS_TYPE_16: + return SDS_HDR(16,s)->len; + case SDS_TYPE_32: + return SDS_HDR(32,s)->len; + case SDS_TYPE_64: + return SDS_HDR(64,s)->len; + } + return 0; } static inline size_t sdsavail(const sds s) { - struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); - return sh->free; + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: { + return 0; + } + case SDS_TYPE_8: { + SDS_HDR_VAR(8,s); + return sh->alloc - sh->len; + } + case SDS_TYPE_16: { + SDS_HDR_VAR(16,s); + return sh->alloc - sh->len; + } + case SDS_TYPE_32: { + SDS_HDR_VAR(32,s); + return sh->alloc - sh->len; + } + case SDS_TYPE_64: { + SDS_HDR_VAR(64,s); + return sh->alloc - sh->len; + } + } + return 0; +} + +static inline void sdssetlen(sds s, size_t newlen) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + { + unsigned char *fp = ((unsigned char*)s)-1; + *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + } + break; + case SDS_TYPE_8: + SDS_HDR(8,s)->len = newlen; + break; + case SDS_TYPE_16: + SDS_HDR(16,s)->len = newlen; + break; + case SDS_TYPE_32: + SDS_HDR(32,s)->len = newlen; + break; + case SDS_TYPE_64: + SDS_HDR(64,s)->len = newlen; + break; + } +} + +static inline void sdsinclen(sds s, size_t inc) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + { + unsigned char *fp = ((unsigned char*)s)-1; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + } + break; + case SDS_TYPE_8: + SDS_HDR(8,s)->len += inc; + break; + case SDS_TYPE_16: + SDS_HDR(16,s)->len += inc; + break; + case SDS_TYPE_32: + SDS_HDR(32,s)->len += inc; + break; + case SDS_TYPE_64: + SDS_HDR(64,s)->len += inc; + break; + } +} + +/* sdsalloc() = sdsavail() + sdslen() */ +static inline size_t sdsalloc(const sds s) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + return SDS_TYPE_5_LEN(flags); + case SDS_TYPE_8: + return SDS_HDR(8,s)->alloc; + case SDS_TYPE_16: + return SDS_HDR(16,s)->alloc; + case SDS_TYPE_32: + return SDS_HDR(32,s)->alloc; + case SDS_TYPE_64: + return SDS_HDR(64,s)->alloc; + } + return 0; +} + +static inline void sdssetalloc(sds s, size_t newlen) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + /* Nothing to do, this type has no total allocation info. */ + break; + case SDS_TYPE_8: + SDS_HDR(8,s)->alloc = newlen; + break; + case SDS_TYPE_16: + SDS_HDR(16,s)->alloc = newlen; + break; + case SDS_TYPE_32: + SDS_HDR(32,s)->alloc = newlen; + break; + case SDS_TYPE_64: + SDS_HDR(64,s)->alloc = newlen; + break; + } } sds sdsnewlen(const void *init, size_t initlen); sds sdsnew(const char *init); sds sdsempty(void); -size_t sdslen(const sds s); sds sdsdup(const sds s); void sdsfree(sds s); -size_t sdsavail(sds s); sds sdsgrowzero(sds s, size_t len); sds sdscatlen(sds s, const void *t, size_t len); sds sdscat(sds s, const char *t); -sds sdscpylen(sds s, char *t, size_t len); -sds sdscpy(sds s, char *t); +sds sdscatsds(sds s, const sds t); +sds sdscpylen(sds s, const char *t, size_t len); +sds sdscpy(sds s, const char *t); sds sdscatvprintf(sds s, const char *fmt, va_list ap); #ifdef __GNUC__ @@ -73,16 +234,40 @@ sds sdscatprintf(sds s, const char *fmt, ...) sds sdscatprintf(sds s, const char *fmt, ...); #endif +sds sdscatfmt(sds s, char const *fmt, ...); sds sdstrim(sds s, const char *cset); -sds sdsrange(sds s, int start, int end); +void sdsrange(sds s, int start, int end); void sdsupdatelen(sds s); -int sdscmp(sds s1, sds s2); -sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count); +void sdsclear(sds s); +int sdscmp(const sds s1, const sds s2); +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); void sdsfreesplitres(sds *tokens, int count); void sdstolower(sds s); void sdstoupper(sds s); sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, char *p, size_t len); -sds *sdssplitargs(char *line, int *argc); +sds sdscatrepr(sds s, const char *p, size_t len); +sds *sdssplitargs(const char *line, int *argc); +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); +sds sdsjoin(char **argv, int argc, char *sep); +sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); + +/* Low level functions exposed to the user API */ +sds sdsMakeRoomFor(sds s, size_t addlen); +void sdsIncrLen(sds s, int incr); +sds sdsRemoveFreeSpace(sds s); +size_t sdsAllocSize(sds s); +void *sdsAllocPtr(sds s); + +/* Export the allocator used by SDS to the program using SDS. + * Sometimes the program SDS is linked to, may use a different set of + * allocators, but may want to allocate or free things that SDS will + * respectively free or allocate. */ +void *sds_malloc(size_t size); +void *sds_realloc(void *ptr, size_t size); +void sds_free(void *ptr); + +#ifdef REDIS_TEST +int sdsTest(int argc, char *argv[]); +#endif #endif diff --git a/deps/hiredis/sdsalloc.h b/deps/hiredis/sdsalloc.h new file mode 100644 index 000000000..f43023c48 --- /dev/null +++ b/deps/hiredis/sdsalloc.h @@ -0,0 +1,42 @@ +/* SDSLib 2.0 -- A C dynamic strings library + * + * Copyright (c) 2006-2015, Salvatore Sanfilippo + * Copyright (c) 2015, Oran Agra + * Copyright (c) 2015, Redis Labs, Inc + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* SDS allocator selection. + * + * This file is used in order to change the SDS allocator at compile time. + * Just define the following defines to what you want to use. Also add + * the include of your alternate allocator if needed (not needed in order + * to use the default libc allocator). */ + +#define s_malloc malloc +#define s_realloc realloc +#define s_free free diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 5945b6552..a23d60676 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -8,12 +8,15 @@ #include #include #include +#include #include "hiredis.h" +#include "net.h" enum connection_type { CONN_TCP, - CONN_UNIX + CONN_UNIX, + CONN_FD }; struct config { @@ -22,11 +25,12 @@ struct config { struct { const char *host; int port; + struct timeval timeout; } tcp; struct { const char *path; - } unix; + } unix_sock; }; /* The following lines make up our testing "framework" :) */ @@ -40,6 +44,13 @@ static long long usec(void) { return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; } +/* The assert() calls below have side effects, so we need assert() + * even if we are compiling without asserts (-DNDEBUG). */ +#ifdef NDEBUG +#undef assert +#define assert(e) (void)(e) +#endif + static redisContext *select_database(redisContext *c) { redisReply *reply; @@ -62,7 +73,7 @@ static redisContext *select_database(redisContext *c) { return c; } -static void disconnect(redisContext *c) { +static int disconnect(redisContext *c, int keep_fd) { redisReply *reply; /* Make sure we're on DB 9. */ @@ -73,8 +84,11 @@ static void disconnect(redisContext *c) { assert(reply != NULL); freeReplyObject(reply); - /* Free the context as well. */ + /* Free the context as well, but keep the fd if requested. */ + if (keep_fd) + return redisFreeKeepFd(c); redisFree(c); + return -1; } static redisContext *connect(struct config config) { @@ -83,13 +97,25 @@ static redisContext *connect(struct config config) { if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); } else if (config.type == CONN_UNIX) { - c = redisConnectUnix(config.unix.path); + c = redisConnectUnix(config.unix_sock.path); + } else if (config.type == CONN_FD) { + /* Create a dummy connection just to get an fd to inherit */ + redisContext *dummy_ctx = redisConnectUnix(config.unix_sock.path); + if (dummy_ctx) { + int fd = disconnect(dummy_ctx, 1); + printf("Connecting to inherited fd %d\n", fd); + c = redisConnectFd(fd); + } } else { assert(NULL); } - if (c->err) { + if (c == NULL) { + printf("Connection error: can't allocate redis context\n"); + exit(1); + } else if (c->err) { printf("Connection error: %s\n", c->errstr); + redisFree(c); exit(1); } @@ -125,13 +151,13 @@ static void test_format_commands(void) { free(cmd); test("Format command with %%b string interpolation: "); - len = redisFormatCommand(&cmd,"SET %b %b","foo",3,"b\0r",3); + len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); free(cmd); test("Format command with %%b and an empty string: "); - len = redisFormatCommand(&cmd,"SET %b %b","foo",3,"",0); + len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); free(cmd); @@ -177,7 +203,7 @@ static void test_format_commands(void) { FLOAT_WIDTH_TEST(double); test("Format command with invalid printf format: "); - len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",3); + len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",(size_t)3); test_cond(len == -1); const char *argv[3]; @@ -198,12 +224,51 @@ static void test_format_commands(void) { test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(7+2)+4+(3+2)); free(cmd); + + sds sds_cmd; + + sds_cmd = sdsempty(); + test("Format command into sds by passing argc/argv without lengths: "); + len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL); + test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && + len == 4+4+(3+2)+4+(3+2)+4+(3+2)); + sdsfree(sds_cmd); + + sds_cmd = sdsempty(); + test("Format command into sds by passing argc/argv with lengths: "); + len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens); + test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && + len == 4+4+(3+2)+4+(7+2)+4+(3+2)); + sdsfree(sds_cmd); +} + +static void test_append_formatted_commands(struct config config) { + redisContext *c; + redisReply *reply; + char *cmd; + int len; + + c = connect(config); + + test("Append format command: "); + + len = redisFormatCommand(&cmd, "SET foo bar"); + + test_cond(redisAppendFormattedCommand(c, cmd, len) == REDIS_OK); + + assert(redisGetReply(c, (void*)&reply) == REDIS_OK); + + free(cmd); + freeReplyObject(reply); + + disconnect(c, 0); } static void test_reply_reader(void) { redisReader *reader; void *reply; int ret; + int i; test("Error handling in reply parser: "); reader = redisReaderCreate(); @@ -225,12 +290,13 @@ static void test_reply_reader(void) { strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); redisReaderFree(reader); - test("Set error on nested multi bulks with depth > 2: "); + test("Set error on nested multi bulks with depth > 7: "); reader = redisReaderCreate(); - redisReaderFeed(reader,(char*)"*1\r\n",4); - redisReaderFeed(reader,(char*)"*1\r\n",4); - redisReaderFeed(reader,(char*)"*1\r\n",4); - redisReaderFeed(reader,(char*)"*1\r\n",4); + + for (i = 0; i < 9; i++) { + redisReaderFeed(reader,(char*)"*1\r\n",4); + } + ret = redisReaderGetReply(reader,NULL); test_cond(ret == REDIS_ERR && strncasecmp(reader->errstr,"No support for",14) == 0); @@ -277,14 +343,32 @@ static void test_reply_reader(void) { redisReaderFree(reader); } +static void test_free_null(void) { + void *redisCtx = NULL; + void *reply = NULL; + + test("Don't fail when redisFree is passed a NULL value: "); + redisFree(redisCtx); + test_cond(redisCtx == NULL); + + test("Don't fail when freeReplyObject is passed a NULL value: "); + freeReplyObject(reply); + test_cond(reply == NULL); +} + static void test_blocking_connection_errors(void) { redisContext *c; test("Returns error when host cannot be resolved: "); - c = redisConnect((char*)"idontexist.local", 6379); + c = redisConnect((char*)"idontexist.test", 6379); test_cond(c->err == REDIS_ERR_OTHER && (strcmp(c->errstr,"Name or service not known") == 0 || - strcmp(c->errstr,"Can't resolve: idontexist.local") == 0)); + strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 || + strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 || + strcmp(c->errstr,"No address associated with hostname") == 0 || + strcmp(c->errstr,"Temporary failure in name resolution") == 0 || + strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 || + strcmp(c->errstr,"no address associated with name") == 0)); redisFree(c); test("Returns error when the port is not open: "); @@ -293,7 +377,7 @@ static void test_blocking_connection_errors(void) { strcmp(c->errstr,"Connection refused") == 0); redisFree(c); - test("Returns error when the unix socket path doesn't accept connections: "); + test("Returns error when the unix_sock socket path doesn't accept connections: "); c = redisConnectUnix((char*)"/tmp/idontexist.sock"); test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */ redisFree(c); @@ -326,7 +410,7 @@ static void test_blocking_connection(struct config config) { freeReplyObject(reply); test("%%b String interpolation works: "); - reply = redisCommand(c,"SET %b %b","foo",3,"hello\x00world",11); + reply = redisCommand(c,"SET %b %b","foo",(size_t)3,"hello\x00world",(size_t)11); freeReplyObject(reply); reply = redisCommand(c,"GET foo"); test_cond(reply->type == REDIS_REPLY_STRING && @@ -374,7 +458,53 @@ static void test_blocking_connection(struct config config) { strcasecmp(reply->element[1]->str,"pong") == 0); freeReplyObject(reply); - disconnect(c); + disconnect(c, 0); +} + +static void test_blocking_connection_timeouts(struct config config) { + redisContext *c; + redisReply *reply; + ssize_t s; + const char *cmd = "DEBUG SLEEP 3\r\n"; + struct timeval tv; + + c = connect(config); + test("Successfully completes a command when the timeout is not exceeded: "); + reply = redisCommand(c,"SET foo fast"); + freeReplyObject(reply); + tv.tv_sec = 0; + tv.tv_usec = 10000; + redisSetTimeout(c, tv); + reply = redisCommand(c, "GET foo"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0); + freeReplyObject(reply); + disconnect(c, 0); + + c = connect(config); + test("Does not return a reply when the command times out: "); + s = write(c->fd, cmd, strlen(cmd)); + tv.tv_sec = 0; + tv.tv_usec = 10000; + redisSetTimeout(c, tv); + reply = redisCommand(c, "GET foo"); + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0); + freeReplyObject(reply); + + test("Reconnect properly reconnects after a timeout: "); + redisReconnect(c); + reply = redisCommand(c, "PING"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); + freeReplyObject(reply); + + test("Reconnect properly uses owned parameters: "); + config.tcp.host = "foo"; + config.unix_sock.path = "foo"; + redisReconnect(c); + reply = redisCommand(c, "PING"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); + freeReplyObject(reply); + + disconnect(c, 0); } static void test_blocking_io_errors(struct config config) { @@ -400,7 +530,7 @@ static void test_blocking_io_errors(struct config config) { test("Returns I/O error when the connection is lost: "); reply = redisCommand(c,"QUIT"); - if (major >= 2 && minor > 0) { + if (major > 2 || (major == 2 && minor > 0)) { /* > 2.0 returns OK on QUIT and read() should be issued once more * to know the descriptor is at EOF. */ test_cond(strcasecmp(reply->str,"OK") == 0 && @@ -428,6 +558,30 @@ static void test_blocking_io_errors(struct config config) { redisFree(c); } +static void test_invalid_timeout_errors(struct config config) { + redisContext *c; + + test("Set error when an invalid timeout usec value is given to redisConnectWithTimeout: "); + + config.tcp.timeout.tv_sec = 0; + config.tcp.timeout.tv_usec = 10000001; + + c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); + + test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0); + redisFree(c); + + test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: "); + + config.tcp.timeout.tv_sec = (((LONG_MAX) - 999) / 1000) + 1; + config.tcp.timeout.tv_usec = 0; + + c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); + + test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0); + redisFree(c); +} + static void test_throughput(struct config config) { redisContext *c = connect(config); redisReply **replies; @@ -490,7 +644,7 @@ static void test_throughput(struct config config) { free(replies); printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - disconnect(c); + disconnect(c, 0); } // static long __test_callback_flags = 0; @@ -598,11 +752,12 @@ int main(int argc, char **argv) { .host = "127.0.0.1", .port = 6379 }, - .unix = { + .unix_sock = { .path = "/tmp/redis.sock" } }; int throughput = 1; + int test_inherit_fd = 1; /* Ignore broken pipe signal (for I/O error tests). */ signal(SIGPIPE, SIG_IGN); @@ -618,9 +773,11 @@ int main(int argc, char **argv) { cfg.tcp.port = atoi(argv[0]); } else if (argc >= 2 && !strcmp(argv[0],"-s")) { argv++; argc--; - cfg.unix.path = argv[0]; + cfg.unix_sock.path = argv[0]; } else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) { throughput = 0; + } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { + test_inherit_fd = 0; } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -631,18 +788,30 @@ int main(int argc, char **argv) { test_format_commands(); test_reply_reader(); test_blocking_connection_errors(); + test_free_null(); printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port); cfg.type = CONN_TCP; test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + + printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path); + cfg.type = CONN_UNIX; + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); - printf("\nTesting against Unix socket connection (%s):\n", cfg.unix.path); - cfg.type = CONN_UNIX; - test_blocking_connection(cfg); - test_blocking_io_errors(cfg); - if (throughput) test_throughput(cfg); + if (test_inherit_fd) { + printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); + cfg.type = CONN_FD; + test_blocking_connection(cfg); + } + if (fails) { printf("*** %d TESTS FAILED ***\n", fails); diff --git a/deps/hiredis/win32.h b/deps/hiredis/win32.h new file mode 100644 index 000000000..1a27c18f2 --- /dev/null +++ b/deps/hiredis/win32.h @@ -0,0 +1,42 @@ +#ifndef _WIN32_HELPER_INCLUDE +#define _WIN32_HELPER_INCLUDE +#ifdef _MSC_VER + +#ifndef inline +#define inline __inline +#endif + +#ifndef va_copy +#define va_copy(d,s) ((d) = (s)) +#endif + +#ifndef snprintf +#define snprintf c99_snprintf + +__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) +{ + int count = -1; + + if (size != 0) + count = _vsnprintf_s(str, size, _TRUNCATE, format, ap); + if (count == -1) + count = _vscprintf(format, ap); + + return count; +} + +__inline int c99_snprintf(char* str, size_t size, const char* format, ...) +{ + int count; + va_list ap; + + va_start(ap, format); + count = c99_vsnprintf(str, size, format, ap); + va_end(ap); + + return count; +} +#endif + +#endif +#endif \ No newline at end of file diff --git a/deps/jemalloc/.appveyor.yml b/deps/jemalloc/.appveyor.yml new file mode 100644 index 000000000..ddd5c5711 --- /dev/null +++ b/deps/jemalloc/.appveyor.yml @@ -0,0 +1,28 @@ +version: '{build}' + +environment: + matrix: + - MSYSTEM: MINGW64 + CPU: x86_64 + MSVC: amd64 + - MSYSTEM: MINGW32 + CPU: i686 + MSVC: x86 + - MSYSTEM: MINGW64 + CPU: x86_64 + - MSYSTEM: MINGW32 + CPU: i686 + +install: + - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH% + - if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC% + - if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc + - pacman --noconfirm -Suy mingw-w64-%CPU%-make + +build_script: + - bash -c "autoconf" + - bash -c "./configure" + - mingw32-make -j3 + - file lib/jemalloc.dll + - mingw32-make -j3 tests + - mingw32-make -k check diff --git a/deps/jemalloc/.autom4te.cfg b/deps/jemalloc/.autom4te.cfg new file mode 100644 index 000000000..fe2424db5 --- /dev/null +++ b/deps/jemalloc/.autom4te.cfg @@ -0,0 +1,3 @@ +begin-language: "Autoconf-without-aclocal-m4" +args: --no-cache +end-language: "Autoconf-without-aclocal-m4" diff --git a/deps/jemalloc/.gitattributes b/deps/jemalloc/.gitattributes new file mode 100644 index 000000000..6313b56c5 --- /dev/null +++ b/deps/jemalloc/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/deps/jemalloc/.gitignore b/deps/jemalloc/.gitignore index 0406413a0..08278d087 100644 --- a/deps/jemalloc/.gitignore +++ b/deps/jemalloc/.gitignore @@ -1,20 +1,91 @@ -/autom4te.cache/ +/*.gcov.* + +/bin/jemalloc-config +/bin/jemalloc.sh +/bin/jeprof + /config.stamp /config.log /config.status +/configure + /doc/html.xsl /doc/manpages.xsl /doc/jemalloc.xml +/doc/jemalloc.html +/doc/jemalloc.3 + +/jemalloc.pc + /lib/ + /Makefile -/include/jemalloc/internal/jemalloc_internal\.h -/include/jemalloc/internal/size_classes\.h -/include/jemalloc/jemalloc\.h -/include/jemalloc/jemalloc_defs\.h + +/include/jemalloc/internal/jemalloc_internal.h +/include/jemalloc/internal/jemalloc_internal_defs.h +/include/jemalloc/internal/private_namespace.h +/include/jemalloc/internal/private_unnamespace.h +/include/jemalloc/internal/public_namespace.h +/include/jemalloc/internal/public_symbols.txt +/include/jemalloc/internal/public_unnamespace.h +/include/jemalloc/internal/size_classes.h +/include/jemalloc/jemalloc.h +/include/jemalloc/jemalloc_defs.h +/include/jemalloc/jemalloc_macros.h +/include/jemalloc/jemalloc_mangle.h +/include/jemalloc/jemalloc_mangle_jet.h +/include/jemalloc/jemalloc_protos.h +/include/jemalloc/jemalloc_protos_jet.h +/include/jemalloc/jemalloc_rename.h +/include/jemalloc/jemalloc_typedefs.h + /src/*.[od] -/test/*.[od] -/test/*.out -/test/[a-zA-Z_]* -!test/*.c -!test/*.exp -/bin/jemalloc.sh +/src/*.gcda +/src/*.gcno + +/test/test.sh +test/include/test/jemalloc_test.h +test/include/test/jemalloc_test_defs.h + +/test/integration/[A-Za-z]* +!/test/integration/[A-Za-z]*.* +/test/integration/*.[od] +/test/integration/*.gcda +/test/integration/*.gcno +/test/integration/*.out + +/test/src/*.[od] +/test/src/*.gcda +/test/src/*.gcno + +/test/stress/[A-Za-z]* +!/test/stress/[A-Za-z]*.* +/test/stress/*.[od] +/test/stress/*.gcda +/test/stress/*.gcno +/test/stress/*.out + +/test/unit/[A-Za-z]* +!/test/unit/[A-Za-z]*.* +/test/unit/*.[od] +/test/unit/*.gcda +/test/unit/*.gcno +/test/unit/*.out + +/VERSION + +*.pdb +*.sdf +*.opendb +*.opensdf +*.cachefile +*.suo +*.user +*.sln.docstates +*.tmp +/msvc/Win32/ +/msvc/x64/ +/msvc/projects/*/*/Debug*/ +/msvc/projects/*/*/Release*/ +/msvc/projects/*/*/Win32/ +/msvc/projects/*/*/x64/ diff --git a/deps/jemalloc/.travis.yml b/deps/jemalloc/.travis.yml new file mode 100644 index 000000000..1fed4f8e6 --- /dev/null +++ b/deps/jemalloc/.travis.yml @@ -0,0 +1,29 @@ +language: c + +matrix: + include: + - os: linux + compiler: gcc + - os: linux + compiler: gcc + env: + - EXTRA_FLAGS=-m32 + addons: + apt: + packages: + - gcc-multilib + - os: osx + compiler: clang + - os: osx + compiler: clang + env: + - EXTRA_FLAGS=-m32 + +before_script: + - autoconf + - ./configure${EXTRA_FLAGS:+ CC="$CC $EXTRA_FLAGS"} + - make -j3 + - make -j3 tests + +script: + - make check diff --git a/deps/jemalloc/COPYING b/deps/jemalloc/COPYING index e27fc4d6c..104b1f8b0 100644 --- a/deps/jemalloc/COPYING +++ b/deps/jemalloc/COPYING @@ -1,10 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2012 Jason Evans . +Copyright (C) 2002-2016 Jason Evans . All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2012 Facebook, Inc. All rights reserved. +Copyright (C) 2009-2016 Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/deps/jemalloc/ChangeLog b/deps/jemalloc/ChangeLog index ab3476c69..f75edd933 100644 --- a/deps/jemalloc/ChangeLog +++ b/deps/jemalloc/ChangeLog @@ -1,10 +1,638 @@ Following are change highlights associated with official releases. Important -bug fixes are all mentioned, but internal enhancements are omitted here for -brevity (even though they are more fun to write about). Much more detail can be -found in the git revision history: +bug fixes are all mentioned, but some internal enhancements are omitted here for +brevity. Much more detail can be found in the git revision history: - http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git - git://canonware.com/jemalloc.git + https://github.com/jemalloc/jemalloc + +* 4.4.0 (December 3, 2016) + + New features: + - Add configure support for *-*-linux-android. (@cferris1000, @jasone) + - Add the --disable-syscall configure option, for use on systems that place + security-motivated limitations on syscall(2). (@jasone) + - Add support for Debian GNU/kFreeBSD. (@thesam) + + Optimizations: + - Add extent serial numbers and use them where appropriate as a sort key that + is higher priority than address, so that the allocation policy prefers older + extents. This tends to improve locality (decrease fragmentation) when + memory grows downward. (@jasone) + - Refactor madvise(2) configuration so that MADV_FREE is detected and utilized + on Linux 4.5 and newer. (@jasone) + - Mark partially purged arena chunks as non-huge-page. This improves + interaction with Linux's transparent huge page functionality. (@jasone) + + Bug fixes: + - Fix size class computations for edge conditions involving extremely large + allocations. This regression was first released in 4.0.0. (@jasone, + @ingvarha) + - Remove overly restrictive assertions related to the cactive statistic. This + regression was first released in 4.1.0. (@jasone) + - Implement a more reliable detection scheme for os_unfair_lock on macOS. + (@jszakmeister) + +* 4.3.1 (November 7, 2016) + + Bug fixes: + - Fix a severe virtual memory leak. This regression was first released in + 4.3.0. (@interwq, @jasone) + - Refactor atomic and prng APIs to restore support for 32-bit platforms that + use pre-C11 toolchains, e.g. FreeBSD's mips. (@jasone) + +* 4.3.0 (November 4, 2016) + + This is the first release that passes the test suite for multiple Windows + configurations, thanks in large part to @glandium setting up continuous + integration via AppVeyor (and Travis CI for Linux and OS X). + + New features: + - Add "J" (JSON) support to malloc_stats_print(). (@jasone) + - Add Cray compiler support. (@ronawho) + + Optimizations: + - Add/use adaptive spinning for bootstrapping and radix tree node + initialization. (@jasone) + + Bug fixes: + - Fix large allocation to search starting in the optimal size class heap, + which can substantially reduce virtual memory churn and fragmentation. This + regression was first released in 4.0.0. (@mjp41, @jasone) + - Fix stats.arenas..nthreads accounting. (@interwq) + - Fix and simplify decay-based purging. (@jasone) + - Make DSS (sbrk(2)-related) operations lockless, which resolves potential + deadlocks during thread exit. (@jasone) + - Fix over-sized allocation of radix tree leaf nodes. (@mjp41, @ogaun, + @jasone) + - Fix over-sized allocation of arena_t (plus associated stats) data + structures. (@jasone, @interwq) + - Fix EXTRA_CFLAGS to not affect configuration. (@jasone) + - Fix a Valgrind integration bug. (@ronawho) + - Disallow 0x5a junk filling when running in Valgrind. (@jasone) + - Fix a file descriptor leak on Linux. This regression was first released in + 4.2.0. (@vsarunas, @jasone) + - Fix static linking of jemalloc with glibc. (@djwatson) + - Use syscall(2) rather than {open,read,close}(2) during boot on Linux. This + works around other libraries' system call wrappers performing reentrant + allocation. (@kspinka, @Whissi, @jasone) + - Fix OS X default zone replacement to work with OS X 10.12. (@glandium, + @jasone) + - Fix cached memory management to avoid needless commit/decommit operations + during purging, which resolves permanent virtual memory map fragmentation + issues on Windows. (@mjp41, @jasone) + - Fix TSD fetches to avoid (recursive) allocation. This is relevant to + non-TLS and Windows configurations. (@jasone) + - Fix malloc_conf overriding to work on Windows. (@jasone) + - Forcibly disable lazy-lock on Windows (was forcibly *enabled*). (@jasone) + +* 4.2.1 (June 8, 2016) + + Bug fixes: + - Fix bootstrapping issues for configurations that require allocation during + tsd initialization (e.g. --disable-tls). (@cferris1000, @jasone) + - Fix gettimeofday() version of nstime_update(). (@ronawho) + - Fix Valgrind regressions in calloc() and chunk_alloc_wrapper(). (@ronawho) + - Fix potential VM map fragmentation regression. (@jasone) + - Fix opt_zero-triggered in-place huge reallocation zeroing. (@jasone) + - Fix heap profiling context leaks in reallocation edge cases. (@jasone) + +* 4.2.0 (May 12, 2016) + + New features: + - Add the arena..reset mallctl, which makes it possible to discard all of + an arena's allocations in a single operation. (@jasone) + - Add the stats.retained and stats.arenas..retained statistics. (@jasone) + - Add the --with-version configure option. (@jasone) + - Support --with-lg-page values larger than actual page size. (@jasone) + + Optimizations: + - Use pairing heaps rather than red-black trees for various hot data + structures. (@djwatson, @jasone) + - Streamline fast paths of rtree operations. (@jasone) + - Optimize the fast paths of calloc() and [m,d,sd]allocx(). (@jasone) + - Decommit unused virtual memory if the OS does not overcommit. (@jasone) + - Specify MAP_NORESERVE on Linux if [heuristic] overcommit is active, in order + to avoid unfortunate interactions during fork(2). (@jasone) + + Bug fixes: + - Fix chunk accounting related to triggering gdump profiles. (@jasone) + - Link against librt for clock_gettime(2) if glibc < 2.17. (@jasone) + - Scale leak report summary according to sampling probability. (@jasone) + +* 4.1.1 (May 3, 2016) + + This bugfix release resolves a variety of mostly minor issues, though the + bitmap fix is critical for 64-bit Windows. + + Bug fixes: + - Fix the linear scan version of bitmap_sfu() to shift by the proper amount + even when sizeof(long) is not the same as sizeof(void *), as on 64-bit + Windows. (@jasone) + - Fix hashing functions to avoid unaligned memory accesses (and resulting + crashes). This is relevant at least to some ARM-based platforms. + (@rkmisra) + - Fix fork()-related lock rank ordering reversals. These reversals were + unlikely to cause deadlocks in practice except when heap profiling was + enabled and active. (@jasone) + - Fix various chunk leaks in OOM code paths. (@jasone) + - Fix malloc_stats_print() to print opt.narenas correctly. (@jasone) + - Fix MSVC-specific build/test issues. (@rustyx, @yuslepukhin) + - Fix a variety of test failures that were due to test fragility rather than + core bugs. (@jasone) + +* 4.1.0 (February 28, 2016) + + This release is primarily about optimizations, but it also incorporates a lot + of portability-motivated refactoring and enhancements. Many people worked on + this release, to an extent that even with the omission here of minor changes + (see git revision history), and of the people who reported and diagnosed + issues, so much of the work was contributed that starting with this release, + changes are annotated with author credits to help reflect the collaborative + effort involved. + + New features: + - Implement decay-based unused dirty page purging, a major optimization with + mallctl API impact. This is an alternative to the existing ratio-based + unused dirty page purging, and is intended to eventually become the sole + purging mechanism. New mallctls: + + opt.purge + + opt.decay_time + + arena..decay + + arena..decay_time + + arenas.decay_time + + stats.arenas..decay_time + (@jasone, @cevans87) + - Add --with-malloc-conf, which makes it possible to embed a default + options string during configuration. This was motivated by the desire to + specify --with-malloc-conf=purge:decay , since the default must remain + purge:ratio until the 5.0.0 release. (@jasone) + - Add MS Visual Studio 2015 support. (@rustyx, @yuslepukhin) + - Make *allocx() size class overflow behavior defined. The maximum + size class is now less than PTRDIFF_MAX to protect applications against + numerical overflow, and all allocation functions are guaranteed to indicate + errors rather than potentially crashing if the request size exceeds the + maximum size class. (@jasone) + - jeprof: + + Add raw heap profile support. (@jasone) + + Add --retain and --exclude for backtrace symbol filtering. (@jasone) + + Optimizations: + - Optimize the fast path to combine various bootstrapping and configuration + checks and execute more streamlined code in the common case. (@interwq) + - Use linear scan for small bitmaps (used for small object tracking). In + addition to speeding up bitmap operations on 64-bit systems, this reduces + allocator metadata overhead by approximately 0.2%. (@djwatson) + - Separate arena_avail trees, which substantially speeds up run tree + operations. (@djwatson) + - Use memoization (boot-time-computed table) for run quantization. Separate + arena_avail trees reduced the importance of this optimization. (@jasone) + - Attempt mmap-based in-place huge reallocation. This can dramatically speed + up incremental huge reallocation. (@jasone) + + Incompatible changes: + - Make opt.narenas unsigned rather than size_t. (@jasone) + + Bug fixes: + - Fix stats.cactive accounting regression. (@rustyx, @jasone) + - Handle unaligned keys in hash(). This caused problems for some ARM systems. + (@jasone, @cferris1000) + - Refactor arenas array. In addition to fixing a fork-related deadlock, this + makes arena lookups faster and simpler. (@jasone) + - Move retained memory allocation out of the default chunk allocation + function, to a location that gets executed even if the application installs + a custom chunk allocation function. This resolves a virtual memory leak. + (@buchgr) + - Fix a potential tsd cleanup leak. (@cferris1000, @jasone) + - Fix run quantization. In practice this bug had no impact unless + applications requested memory with alignment exceeding one page. + (@jasone, @djwatson) + - Fix LinuxThreads-specific bootstrapping deadlock. (Cosmin Paraschiv) + - jeprof: + + Don't discard curl options if timeout is not defined. (@djwatson) + + Detect failed profile fetches. (@djwatson) + - Fix stats.arenas..{dss,lg_dirty_mult,decay_time,pactive,pdirty} for + --disable-stats case. (@jasone) + +* 4.0.4 (October 24, 2015) + + This bugfix release fixes another xallocx() regression. No other regressions + have come to light in over a month, so this is likely a good starting point + for people who prefer to wait for "dot one" releases with all the major issues + shaken out. + + Bug fixes: + - Fix xallocx(..., MALLOCX_ZERO to zero the last full trailing page of large + allocations that have been randomly assigned an offset of 0 when + --enable-cache-oblivious configure option is enabled. + +* 4.0.3 (September 24, 2015) + + This bugfix release continues the trend of xallocx() and heap profiling fixes. + + Bug fixes: + - Fix xallocx(..., MALLOCX_ZERO) to zero all trailing bytes of large + allocations when --enable-cache-oblivious configure option is enabled. + - Fix xallocx(..., MALLOCX_ZERO) to zero trailing bytes of huge allocations + when resizing from/to a size class that is not a multiple of the chunk size. + - Fix prof_tctx_dump_iter() to filter out nodes that were created after heap + profile dumping started. + - Work around a potentially bad thread-specific data initialization + interaction with NPTL (glibc's pthreads implementation). + +* 4.0.2 (September 21, 2015) + + This bugfix release addresses a few bugs specific to heap profiling. + + Bug fixes: + - Fix ixallocx_prof_sample() to never modify nor create sampled small + allocations. xallocx() is in general incapable of moving small allocations, + so this fix removes buggy code without loss of generality. + - Fix irallocx_prof_sample() to always allocate large regions, even when + alignment is non-zero. + - Fix prof_alloc_rollback() to read tdata from thread-specific data rather + than dereferencing a potentially invalid tctx. + +* 4.0.1 (September 15, 2015) + + This is a bugfix release that is somewhat high risk due to the amount of + refactoring required to address deep xallocx() problems. As a side effect of + these fixes, xallocx() now tries harder to partially fulfill requests for + optional extra space. Note that a couple of minor heap profiling + optimizations are included, but these are better thought of as performance + fixes that were integral to disovering most of the other bugs. + + Optimizations: + - Avoid a chunk metadata read in arena_prof_tctx_set(), since it is in the + fast path when heap profiling is enabled. Additionally, split a special + case out into arena_prof_tctx_reset(), which also avoids chunk metadata + reads. + - Optimize irallocx_prof() to optimistically update the sampler state. The + prior implementation appears to have been a holdover from when + rallocx()/xallocx() functionality was combined as rallocm(). + + Bug fixes: + - Fix TLS configuration such that it is enabled by default for platforms on + which it works correctly. + - Fix arenas_cache_cleanup() and arena_get_hard() to handle + allocation/deallocation within the application's thread-specific data + cleanup functions even after arenas_cache is torn down. + - Fix xallocx() bugs related to size+extra exceeding HUGE_MAXCLASS. + - Fix chunk purge hook calls for in-place huge shrinking reallocation to + specify the old chunk size rather than the new chunk size. This bug caused + no correctness issues for the default chunk purge function, but was + visible to custom functions set via the "arena..chunk_hooks" mallctl. + - Fix heap profiling bugs: + + Fix heap profiling to distinguish among otherwise identical sample sites + with interposed resets (triggered via the "prof.reset" mallctl). This bug + could cause data structure corruption that would most likely result in a + segfault. + + Fix irealloc_prof() to prof_alloc_rollback() on OOM. + + Make one call to prof_active_get_unlocked() per allocation event, and use + the result throughout the relevant functions that handle an allocation + event. Also add a missing check in prof_realloc(). These fixes protect + allocation events against concurrent prof_active changes. + + Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample() + in the correct order. + + Fix prof_realloc() to call prof_free_sampled_object() after calling + prof_malloc_sample_object(). Prior to this fix, if tctx and old_tctx were + the same, the tctx could have been prematurely destroyed. + - Fix portability bugs: + + Don't bitshift by negative amounts when encoding/decoding run sizes in + chunk header maps. This affected systems with page sizes greater than 8 + KiB. + + Rename index_t to szind_t to avoid an existing type on Solaris. + + Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to + match glibc and avoid compilation errors when including both + jemalloc/jemalloc.h and malloc.h in C++ code. + + Don't assume that /bin/sh is appropriate when running size_classes.sh + during configuration. + + Consider __sparcv9 a synonym for __sparc64__ when defining LG_QUANTUM. + + Link tests to librt if it contains clock_gettime(2). + +* 4.0.0 (August 17, 2015) + + This version contains many speed and space optimizations, both minor and + major. The major themes are generalization, unification, and simplification. + Although many of these optimizations cause no visible behavior change, their + cumulative effect is substantial. + + New features: + - Normalize size class spacing to be consistent across the complete size + range. By default there are four size classes per size doubling, but this + is now configurable via the --with-lg-size-class-group option. Also add the + --with-lg-page, --with-lg-page-sizes, --with-lg-quantum, and + --with-lg-tiny-min options, which can be used to tweak page and size class + settings. Impacts: + + Worst case performance for incrementally growing/shrinking reallocation + is improved because there are far fewer size classes, and therefore + copying happens less often. + + Internal fragmentation is limited to 20% for all but the smallest size + classes (those less than four times the quantum). (1B + 4 KiB) + and (1B + 4 MiB) previously suffered nearly 50% internal fragmentation. + + Chunk fragmentation tends to be lower because there are fewer distinct run + sizes to pack. + - Add support for explicit tcaches. The "tcache.create", "tcache.flush", and + "tcache.destroy" mallctls control tcache lifetime and flushing, and the + MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to the *allocx() API + control which tcache is used for each operation. + - Implement per thread heap profiling, as well as the ability to + enable/disable heap profiling on a per thread basis. Add the "prof.reset", + "prof.lg_sample", "thread.prof.name", "thread.prof.active", + "opt.prof_thread_active_init", "prof.thread_active_init", and + "thread.prof.active" mallctls. + - Add support for per arena application-specified chunk allocators, configured + via the "arena..chunk_hooks" mallctl. + - Refactor huge allocation to be managed by arenas, so that arenas now + function as general purpose independent allocators. This is important in + the context of user-specified chunk allocators, aside from the scalability + benefits. Related new statistics: + + The "stats.arenas..huge.allocated", "stats.arenas..huge.nmalloc", + "stats.arenas..huge.ndalloc", and "stats.arenas..huge.nrequests" + mallctls provide high level per arena huge allocation statistics. + + The "arenas.nhchunks", "arenas.hchunk..size", + "stats.arenas..hchunks..nmalloc", + "stats.arenas..hchunks..ndalloc", + "stats.arenas..hchunks..nrequests", and + "stats.arenas..hchunks..curhchunks" mallctls provide per size class + statistics. + - Add the 'util' column to malloc_stats_print() output, which reports the + proportion of available regions that are currently in use for each small + size class. + - Add "alloc" and "free" modes for for junk filling (see the "opt.junk" + mallctl), so that it is possible to separately enable junk filling for + allocation versus deallocation. + - Add the jemalloc-config script, which provides information about how + jemalloc was configured, and how to integrate it into application builds. + - Add metadata statistics, which are accessible via the "stats.metadata", + "stats.arenas..metadata.mapped", and + "stats.arenas..metadata.allocated" mallctls. + - Add the "stats.resident" mallctl, which reports the upper limit of + physically resident memory mapped by the allocator. + - Add per arena control over unused dirty page purging, via the + "arenas.lg_dirty_mult", "arena..lg_dirty_mult", and + "stats.arenas..lg_dirty_mult" mallctls. + - Add the "prof.gdump" mallctl, which makes it possible to toggle the gdump + feature on/off during program execution. + - Add sdallocx(), which implements sized deallocation. The primary + optimization over dallocx() is the removal of a metadata read, which often + suffers an L1 cache miss. + - Add missing header includes in jemalloc/jemalloc.h, so that applications + only have to #include . + - Add support for additional platforms: + + Bitrig + + Cygwin + + DragonFlyBSD + + iOS + + OpenBSD + + OpenRISC/or1k + + Optimizations: + - Maintain dirty runs in per arena LRUs rather than in per arena trees of + dirty-run-containing chunks. In practice this change significantly reduces + dirty page purging volume. + - Integrate whole chunks into the unused dirty page purging machinery. This + reduces the cost of repeated huge allocation/deallocation, because it + effectively introduces a cache of chunks. + - Split the arena chunk map into two separate arrays, in order to increase + cache locality for the frequently accessed bits. + - Move small run metadata out of runs, into arena chunk headers. This reduces + run fragmentation, smaller runs reduce external fragmentation for small size + classes, and packed (less uniformly aligned) metadata layout improves CPU + cache set distribution. + - Randomly distribute large allocation base pointer alignment relative to page + boundaries in order to more uniformly utilize CPU cache sets. This can be + disabled via the --disable-cache-oblivious configure option, and queried via + the "config.cache_oblivious" mallctl. + - Micro-optimize the fast paths for the public API functions. + - Refactor thread-specific data to reside in a single structure. This assures + that only a single TLS read is necessary per call into the public API. + - Implement in-place huge allocation growing and shrinking. + - Refactor rtree (radix tree for chunk lookups) to be lock-free, and make + additional optimizations that reduce maximum lookup depth to one or two + levels. This resolves what was a concurrency bottleneck for per arena huge + allocation, because a global data structure is critical for determining + which arenas own which huge allocations. + + Incompatible changes: + - Replace --enable-cc-silence with --disable-cc-silence to suppress spurious + warnings by default. + - Assure that the constness of malloc_usable_size()'s return type matches that + of the system implementation. + - Change the heap profile dump format to support per thread heap profiling, + rename pprof to jeprof, and enhance it with the --thread= option. As a + result, the bundled jeprof must now be used rather than the upstream + (gperftools) pprof. + - Disable "opt.prof_final" by default, in order to avoid atexit(3), which can + internally deadlock on some platforms. + - Change the "arenas.nlruns" mallctl type from size_t to unsigned. + - Replace the "stats.arenas..bins..allocated" mallctl with + "stats.arenas..bins..curregs". + - Ignore MALLOC_CONF in set{uid,gid,cap} binaries. + - Ignore MALLOCX_ARENA(a) in dallocx(), in favor of using the + MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to control tcache usage. + + Removed features: + - Remove the *allocm() API, which is superseded by the *allocx() API. + - Remove the --enable-dss options, and make dss non-optional on all platforms + which support sbrk(2). + - Remove the "arenas.purge" mallctl, which was obsoleted by the + "arena..purge" mallctl in 3.1.0. + - Remove the unnecessary "opt.valgrind" mallctl; jemalloc automatically + detects whether it is running inside Valgrind. + - Remove the "stats.huge.allocated", "stats.huge.nmalloc", and + "stats.huge.ndalloc" mallctls. + - Remove the --enable-mremap option. + - Remove the "stats.chunks.current", "stats.chunks.total", and + "stats.chunks.high" mallctls. + + Bug fixes: + - Fix the cactive statistic to decrease (rather than increase) when active + memory decreases. This regression was first released in 3.5.0. + - Fix OOM handling in memalign() and valloc(). A variant of this bug existed + in all releases since 2.0.0, which introduced these functions. + - Fix an OOM-related regression in arena_tcache_fill_small(), which could + cause cache corruption on OOM. This regression was present in all releases + from 2.2.0 through 3.6.0. + - Fix size class overflow handling for malloc(), posix_memalign(), memalign(), + calloc(), and realloc() when profiling is enabled. + - Fix the "arena..dss" mallctl to return an error if "primary" or + "secondary" precedence is specified, but sbrk(2) is not supported. + - Fix fallback lg_floor() implementations to handle extremely large inputs. + - Ensure the default purgeable zone is after the default zone on OS X. + - Fix latent bugs in atomic_*(). + - Fix the "arena..dss" mallctl to handle read-only calls. + - Fix tls_model configuration to enable the initial-exec model when possible. + - Mark malloc_conf as a weak symbol so that the application can override it. + - Correctly detect glibc's adaptive pthread mutexes. + - Fix the --without-export configure option. + +* 3.6.0 (March 31, 2014) + + This version contains a critical bug fix for a regression present in 3.5.0 and + 3.5.1. + + Bug fixes: + - Fix a regression in arena_chunk_alloc() that caused crashes during + small/large allocation if chunk allocation failed. In the absence of this + bug, chunk allocation failure would result in allocation failure, e.g. NULL + return from malloc(). This regression was introduced in 3.5.0. + - Fix backtracing for gcc intrinsics-based backtracing by specifying + -fno-omit-frame-pointer to gcc. Note that the application (and all the + libraries it links to) must also be compiled with this option for + backtracing to be reliable. + - Use dss allocation precedence for huge allocations as well as small/large + allocations. + - Fix test assertion failure message formatting. This bug did not manifest on + x86_64 systems because of implementation subtleties in va_list. + - Fix inconsequential test failures for hash and SFMT code. + + New features: + - Support heap profiling on FreeBSD. This feature depends on the proc + filesystem being mounted during heap profile dumping. + +* 3.5.1 (February 25, 2014) + + This version primarily addresses minor bugs in test code. + + Bug fixes: + - Configure Solaris/Illumos to use MADV_FREE. + - Fix junk filling for mremap(2)-based huge reallocation. This is only + relevant if configuring with the --enable-mremap option specified. + - Avoid compilation failure if 'restrict' C99 keyword is not supported by the + compiler. + - Add a configure test for SSE2 rather than assuming it is usable on i686 + systems. This fixes test compilation errors, especially on 32-bit Linux + systems. + - Fix mallctl argument size mismatches (size_t vs. uint64_t) in the stats unit + test. + - Fix/remove flawed alignment-related overflow tests. + - Prevent compiler optimizations that could change backtraces in the + prof_accum unit test. + +* 3.5.0 (January 22, 2014) + + This version focuses on refactoring and automated testing, though it also + includes some non-trivial heap profiling optimizations not mentioned below. + + New features: + - Add the *allocx() API, which is a successor to the experimental *allocm() + API. The *allocx() functions are slightly simpler to use because they have + fewer parameters, they directly return the results of primary interest, and + mallocx()/rallocx() avoid the strict aliasing pitfall that + allocm()/rallocm() share with posix_memalign(). Note that *allocm() is + slated for removal in the next non-bugfix release. + - Add support for LinuxThreads. + + Bug fixes: + - Unless heap profiling is enabled, disable floating point code and don't link + with libm. This, in combination with e.g. EXTRA_CFLAGS=-mno-sse on x64 + systems, makes it possible to completely disable floating point register + use. Some versions of glibc neglect to save/restore caller-saved floating + point registers during dynamic lazy symbol loading, and the symbol loading + code uses whatever malloc the application happens to have linked/loaded + with, the result being potential floating point register corruption. + - Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling + backtrace creation in imemalign(). This bug impacted posix_memalign() and + aligned_alloc(). + - Fix a file descriptor leak in a prof_dump_maps() error path. + - Fix prof_dump() to close the dump file descriptor for all relevant error + paths. + - Fix rallocm() to use the arena specified by the ALLOCM_ARENA(s) flag for + allocation, not just deallocation. + - Fix a data race for large allocation stats counters. + - Fix a potential infinite loop during thread exit. This bug occurred on + Solaris, and could affect other platforms with similar pthreads TSD + implementations. + - Don't junk-fill reallocations unless usable size changes. This fixes a + violation of the *allocx()/*allocm() semantics. + - Fix growing large reallocation to junk fill new space. + - Fix huge deallocation to junk fill when munmap is disabled. + - Change the default private namespace prefix from empty to je_, and change + --with-private-namespace-prefix so that it prepends an additional prefix + rather than replacing je_. This reduces the likelihood of applications + which statically link jemalloc experiencing symbol name collisions. + - Add missing private namespace mangling (relevant when + --with-private-namespace is specified). + - Add and use JEMALLOC_INLINE_C so that static inline functions are marked as + static even for debug builds. + - Add a missing mutex unlock in a malloc_init_hard() error path. In practice + this error path is never executed. + - Fix numerous bugs in malloc_strotumax() error handling/reporting. These + bugs had no impact except for malformed inputs. + - Fix numerous bugs in malloc_snprintf(). These bugs were not exercised by + existing calls, so they had no impact. + +* 3.4.1 (October 20, 2013) + + Bug fixes: + - Fix a race in the "arenas.extend" mallctl that could cause memory corruption + of internal data structures and subsequent crashes. + - Fix Valgrind integration flaws that caused Valgrind warnings about reads of + uninitialized memory in: + + arena chunk headers + + internal zero-initialized data structures (relevant to tcache and prof + code) + - Preserve errno during the first allocation. A readlink(2) call during + initialization fails unless /etc/malloc.conf exists, so errno was typically + set during the first allocation prior to this fix. + - Fix compilation warnings reported by gcc 4.8.1. + +* 3.4.0 (June 2, 2013) + + This version is essentially a small bugfix release, but the addition of + aarch64 support requires that the minor version be incremented. + + Bug fixes: + - Fix race-triggered deadlocks in chunk_record(). These deadlocks were + typically triggered by multiple threads concurrently deallocating huge + objects. + + New features: + - Add support for the aarch64 architecture. + +* 3.3.1 (March 6, 2013) + + This version fixes bugs that are typically encountered only when utilizing + custom run-time options. + + Bug fixes: + - Fix a locking order bug that could cause deadlock during fork if heap + profiling were enabled. + - Fix a chunk recycling bug that could cause the allocator to lose track of + whether a chunk was zeroed. On FreeBSD, NetBSD, and OS X, it could cause + corruption if allocating via sbrk(2) (unlikely unless running with the + "dss:primary" option specified). This was completely harmless on Linux + unless using mlockall(2) (and unlikely even then, unless the + --disable-munmap configure option or the "dss:primary" option was + specified). This regression was introduced in 3.1.0 by the + mlockall(2)/madvise(2) interaction fix. + - Fix TLS-related memory corruption that could occur during thread exit if the + thread never allocated memory. Only the quarantine and prof facilities were + susceptible. + - Fix two quarantine bugs: + + Internal reallocation of the quarantined object array leaked the old + array. + + Reallocation failure for internal reallocation of the quarantined object + array (very unlikely) resulted in memory corruption. + - Fix Valgrind integration to annotate all internally allocated memory in a + way that keeps Valgrind happy about internal data structure access. + - Fix building for s390 systems. + +* 3.3.0 (January 23, 2013) + + This version includes a few minor performance improvements in addition to the + listed new features and bug fixes. + + New features: + - Add clipping support to lg_chunk option processing. + - Add the --enable-ivsalloc option. + - Add the --without-export option. + - Add the --disable-zone-allocator option. + + Bug fixes: + - Fix "arenas.extend" mallctl to output the number of arenas. + - Fix chunk_recycle() to unconditionally inform Valgrind that returned memory + is undefined. + - Fix build break on FreeBSD related to alloca.h. * 3.2.0 (November 9, 2012) @@ -348,7 +976,7 @@ found in the git revision history: - Make it possible for the application to manually flush a thread's cache, via the "tcache.flush" mallctl. - Base maximum dirty page count on proportion of active memory. - - Compute various addtional run-time statistics, including per size class + - Compute various additional run-time statistics, including per size class statistics for large objects. - Expose malloc_stats_print(), which can be called repeatedly by the application. diff --git a/deps/jemalloc/INSTALL b/deps/jemalloc/INSTALL index e40a7eddc..cce3ed711 100644 --- a/deps/jemalloc/INSTALL +++ b/deps/jemalloc/INSTALL @@ -1,10 +1,23 @@ -Building and installing jemalloc can be as simple as typing the following while -in the root directory of the source tree: +Building and installing a packaged release of jemalloc can be as simple as +typing the following while in the root directory of the source tree: ./configure make make install +If building from unpackaged developer sources, the simplest command sequence +that might work is: + + ./autogen.sh + make dist + make + make install + +Note that documentation is not built by the default target because doing so +would create a dependency on xsltproc in packaged releases, hence the +requirement to either run 'make dist' or avoid installing docs via the various +install_* targets documented below. + === Advanced configuration ===================================================== The 'configure' script supports numerous options that allow control of which @@ -22,6 +35,10 @@ any of the following arguments (not a definitive list) to 'configure': will cause files to be installed into /usr/local/include, /usr/local/lib, and /usr/local/man. +--with-version=..--g + Use the specified version string rather than trying to generate one (if in + a git repository) or use existing the VERSION file (if present). + --with-rpath= Embed one or more library paths, so that libjemalloc can find the libraries it is linked to. This works only on ELF-based systems. @@ -55,30 +72,62 @@ any of the following arguments (not a definitive list) to 'configure': jemalloc overlays the default malloc zone, but makes no attempt to actually replace the "malloc", "calloc", etc. symbols. +--without-export + Don't export public APIs. This can be useful when building jemalloc as a + static library, or to avoid exporting public APIs when using the zone + allocator on OSX. + --with-private-namespace= - Prefix all library-private APIs with . For shared libraries, + Prefix all library-private APIs with je_. For shared libraries, symbol visibility mechanisms prevent these symbols from being exported, but for static libraries, naming collisions are a real possibility. By - default, the prefix is "" (empty string). + default, is empty, which results in a symbol prefix of je_ . --with-install-suffix= Append to the base name of all installed files, such that multiple versions of jemalloc can coexist in the same installation directory. For example, libjemalloc.so.0 becomes libjemalloc.so.0. ---enable-cc-silence - Enable code that silences non-useful compiler warnings. This is helpful - when trying to tell serious warnings from those due to compiler - limitations, but it potentially incurs a performance penalty. +--with-malloc-conf= + Embed as a run-time options string that is processed prior to + the malloc_conf global variable, the /etc/malloc.conf symlink, and the + MALLOC_CONF environment variable. For example, to change the default chunk + size to 256 KiB: + + --with-malloc-conf=lg_chunk:18 + +--disable-cc-silence + Disable code that silences non-useful compiler warnings. This is mainly + useful during development when auditing the set of warnings that are being + silenced. --enable-debug Enable assertions and validation code. This incurs a substantial performance hit, but is very useful during application development. + Implies --enable-ivsalloc. + +--enable-code-coverage + Enable code coverage support, for use during jemalloc test development. + Additional testing targets are available if this option is enabled: + + coverage + coverage_unit + coverage_integration + coverage_stress + + These targets do not clear code coverage results from previous runs, and + there are interactions between the various coverage targets, so it is + usually advisable to run 'make clean' between repeated code coverage runs. --disable-stats Disable statistics gathering functionality. See the "opt.stats_print" option documentation for usage details. +--enable-ivsalloc + Enable validation code, which verifies that pointers reside within + jemalloc-owned chunks before dereferencing them. This incurs a minor + performance hit. + --enable-prof Enable heap profiling and leak detection functionality. See the "opt.prof" option documentation for usage details. When enabled, there are several @@ -108,12 +157,6 @@ any of the following arguments (not a definitive list) to 'configure': released in bulk, thus reducing the total number of mutex operations. See the "opt.tcache" option for usage details. ---enable-mremap - Enable huge realloc() via mremap(2). mremap() is disabled by default - because the flavor used is specific to Linux, which has a quirk in its - virtual memory allocation algorithm that causes semi-permanent VM map holes - under normal jemalloc operation. - --disable-munmap Disable virtual memory deallocation via munmap(2); instead keep track of the virtual memory for later use. munmap() is disabled by default (i.e. @@ -121,10 +164,6 @@ any of the following arguments (not a definitive list) to 'configure': memory allocation algorithm that causes semi-permanent VM map holes under normal jemalloc operation. ---enable-dss - Enable support for page allocation/deallocation via sbrk(2), in addition to - mmap(2). - --disable-fill Disable support for junk/zero filling of memory, quarantine, and redzones. See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option @@ -133,8 +172,9 @@ any of the following arguments (not a definitive list) to 'configure': --disable-valgrind Disable support for Valgrind. ---disable-experimental - Disable support for the experimental API (*allocm()). +--disable-zone-allocator + Disable zone allocator for Darwin. This means jemalloc won't be hooked as + the default allocator on OSX/iOS. --enable-utrace Enable utrace(2)-based allocation tracing. This feature is not broadly @@ -157,10 +197,111 @@ any of the following arguments (not a definitive list) to 'configure': thread-local variables via the __thread keyword. If TLS is available, jemalloc uses it for several purposes. +--disable-cache-oblivious + Disable cache-oblivious large allocation alignment for large allocation + requests with no alignment constraints. If this feature is disabled, all + large allocations are page-aligned as an implementation artifact, which can + severely harm CPU cache utilization. However, the cache-oblivious layout + comes at the cost of one extra page per large allocation, which in the + most extreme case increases physical memory usage for the 16 KiB size class + to 20 KiB. + +--disable-syscall + Disable use of syscall(2) rather than {open,read,write,close}(2). This is + intended as a workaround for systems that place security limitations on + syscall(2). + --with-xslroot= Specify where to find DocBook XSL stylesheets when building the documentation. +--with-lg-page= + Specify the base 2 log of the system page size. This option is only useful + when cross compiling, since the configure script automatically determines + the host's page size by default. + +--with-lg-page-sizes= + Specify the comma-separated base 2 logs of the page sizes to support. This + option may be useful when cross-compiling in combination with + --with-lg-page, but its primary use case is for integration with FreeBSD's + libc, wherein jemalloc is embedded. + +--with-lg-size-class-group= + Specify the base 2 log of how many size classes to use for each doubling in + size. By default jemalloc uses =2, which results in + e.g. the following size classes: + + [...], 64, + 80, 96, 112, 128, + 160, [...] + + =3 results in e.g. the following size classes: + + [...], 64, + 72, 80, 88, 96, 104, 112, 120, 128, + 144, [...] + + The minimal =0 causes jemalloc to only provide size + classes that are powers of 2: + + [...], + 64, + 128, + 256, + [...] + + An implementation detail currently limits the total number of small size + classes to 255, and a compilation error will result if the + you specify cannot be supported. The limit is + roughly =4, depending on page size. + +--with-lg-quantum= + Specify the base 2 log of the minimum allocation alignment. jemalloc needs + to know the minimum alignment that meets the following C standard + requirement (quoted from the April 12, 2011 draft of the C11 standard): + + The pointer returned if the allocation succeeds is suitably aligned so + that it may be assigned to a pointer to any type of object with a + fundamental alignment requirement and then used to access such an object + or an array of such objects in the space allocated [...] + + This setting is architecture-specific, and although jemalloc includes known + safe values for the most commonly used modern architectures, there is a + wrinkle related to GNU libc (glibc) that may impact your choice of + . On most modern architectures, this mandates 16-byte alignment + (=4), but the glibc developers chose not to meet this + requirement for performance reasons. An old discussion can be found at + https://sourceware.org/bugzilla/show_bug.cgi?id=206 . Unlike glibc, + jemalloc does follow the C standard by default (caveat: jemalloc + technically cheats if --with-lg-tiny-min is smaller than + --with-lg-quantum), but the fact that Linux systems already work around + this allocator noncompliance means that it is generally safe in practice to + let jemalloc's minimum alignment follow glibc's lead. If you specify + --with-lg-quantum=3 during configuration, jemalloc will provide additional + size classes that are not 16-byte-aligned (24, 40, and 56, assuming + --with-lg-size-class-group=2). + +--with-lg-tiny-min= + Specify the base 2 log of the minimum tiny size class to support. Tiny + size classes are powers of 2 less than the quantum, and are only + incorporated if is less than (see + --with-lg-quantum). Tiny size classes technically violate the C standard + requirement for minimum alignment, and crashes could conceivably result if + the compiler were to generate instructions that made alignment assumptions, + both because illegal instruction traps could result, and because accesses + could straddle page boundaries and cause segmentation faults due to + accessing unmapped addresses. + + The default of =3 works well in practice even on architectures + that technically require 16-byte alignment, probably for the same reason + --with-lg-quantum=3 works. Smaller tiny size classes can, and will, cause + crashes (see https://bugzilla.mozilla.org/show_bug.cgi?id=691003 for an + example). + + This option is rarely useful, and is mainly provided as documentation of a + subtle implementation detail. If you do use this option, specify a + value in [3, ..., ]. + The following environment variables (not a definitive list) impact configure's behavior: @@ -191,6 +332,15 @@ LDFLAGS="?" PATH="?" 'configure' uses this to find programs. +In some cases it may be necessary to work around configuration results that do +not match reality. For example, Linux 4.5 added support for the MADV_FREE flag +to madvise(2), which can cause problems if building on a host with MADV_FREE +support and deploying to a target without. To work around this, use a cache +file to override the relevant configuration variable defined in configure.ac, +e.g.: + + echo "je_cv_madv_free=no" > config.cache && ./configure -C + === Advanced compilation ======================================================= To build only parts of jemalloc, use the following targets: diff --git a/deps/jemalloc/Makefile.in b/deps/jemalloc/Makefile.in index 364481890..c70536391 100644 --- a/deps/jemalloc/Makefile.in +++ b/deps/jemalloc/Makefile.in @@ -24,7 +24,8 @@ abs_objroot := @abs_objroot@ # Build parameters. CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include -CFLAGS := @CFLAGS@ +EXTRA_CFLAGS := @EXTRA_CFLAGS@ +CFLAGS := @CFLAGS@ $(EXTRA_CFLAGS) LDFLAGS := @LDFLAGS@ EXTRA_LDFLAGS := @EXTRA_LDFLAGS@ LIBS := @LIBS@ @@ -42,19 +43,29 @@ XSLTPROC := @XSLTPROC@ AUTOCONF := @AUTOCONF@ _RPATH = @RPATH@ RPATH = $(if $(1),$(call _RPATH,$(1))) -cfghdrs_in := @cfghdrs_in@ +cfghdrs_in := $(addprefix $(srcroot),@cfghdrs_in@) cfghdrs_out := @cfghdrs_out@ -cfgoutputs_in := @cfgoutputs_in@ +cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@) cfgoutputs_out := @cfgoutputs_out@ enable_autogen := @enable_autogen@ -enable_experimental := @enable_experimental@ +enable_code_coverage := @enable_code_coverage@ +enable_prof := @enable_prof@ +enable_valgrind := @enable_valgrind@ +enable_zone_allocator := @enable_zone_allocator@ +MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF +link_whole_archive := @link_whole_archive@ DSO_LDFLAGS = @DSO_LDFLAGS@ SOREV = @SOREV@ PIC_CFLAGS = @PIC_CFLAGS@ CTARGET = @CTARGET@ LDTARGET = @LDTARGET@ +TEST_LD_MODE = @TEST_LD_MODE@ MKLIB = @MKLIB@ +AR = @AR@ +ARFLAGS = @ARFLAGS@ CC_MM = @CC_MM@ +LM := @LM@ +INSTALL = @INSTALL@ ifeq (macho, $(ABI)) TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib" @@ -69,19 +80,41 @@ endif LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix) # Lists of files. -BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh -CHDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h \ - $(objroot)include/jemalloc/jemalloc_defs$(install_suffix).h -CSRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c $(srcroot)src/atomic.c \ - $(srcroot)src/base.c $(srcroot)src/bitmap.c $(srcroot)src/chunk.c \ - $(srcroot)src/chunk_dss.c $(srcroot)src/chunk_mmap.c \ - $(srcroot)src/ckh.c $(srcroot)src/ctl.c $(srcroot)src/extent.c \ - $(srcroot)src/hash.c $(srcroot)src/huge.c $(srcroot)src/mb.c \ - $(srcroot)src/mutex.c $(srcroot)src/prof.c $(srcroot)src/quarantine.c \ - $(srcroot)src/rtree.c $(srcroot)src/stats.c $(srcroot)src/tcache.c \ - $(srcroot)src/util.c $(srcroot)src/tsd.c -ifeq (macho, $(ABI)) -CSRCS += $(srcroot)src/zone.c +BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof +C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h +C_SRCS := $(srcroot)src/jemalloc.c \ + $(srcroot)src/arena.c \ + $(srcroot)src/atomic.c \ + $(srcroot)src/base.c \ + $(srcroot)src/bitmap.c \ + $(srcroot)src/chunk.c \ + $(srcroot)src/chunk_dss.c \ + $(srcroot)src/chunk_mmap.c \ + $(srcroot)src/ckh.c \ + $(srcroot)src/ctl.c \ + $(srcroot)src/extent.c \ + $(srcroot)src/hash.c \ + $(srcroot)src/huge.c \ + $(srcroot)src/mb.c \ + $(srcroot)src/mutex.c \ + $(srcroot)src/nstime.c \ + $(srcroot)src/pages.c \ + $(srcroot)src/prng.c \ + $(srcroot)src/prof.c \ + $(srcroot)src/quarantine.c \ + $(srcroot)src/rtree.c \ + $(srcroot)src/stats.c \ + $(srcroot)src/spin.c \ + $(srcroot)src/tcache.c \ + $(srcroot)src/ticker.c \ + $(srcroot)src/tsd.c \ + $(srcroot)src/util.c \ + $(srcroot)src/witness.c +ifeq ($(enable_valgrind), 1) +C_SRCS += $(srcroot)src/valgrind.c +endif +ifeq ($(enable_zone_allocator), 1) +C_SRCS += $(srcroot)src/zone.c endif ifeq ($(IMPORTLIB),$(SO)) STATIC_LIBS := $(objroot)lib/$(LIBJEMALLOC).$(A) @@ -95,39 +128,115 @@ DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV) ifneq ($(SOREV),$(SO)) DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO) endif +ifeq (1, $(link_whole_archive)) +LJEMALLOC := -Wl,--whole-archive -L$(objroot)lib -l$(LIBJEMALLOC) -Wl,--no-whole-archive +else +LJEMALLOC := $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) +endif +PC := $(objroot)jemalloc.pc MAN3 := $(objroot)doc/jemalloc$(install_suffix).3 DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml -DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) -DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3) +DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.html) +DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.3) DOCS := $(DOCS_HTML) $(DOCS_MAN3) -CTESTS := $(srcroot)test/aligned_alloc.c $(srcroot)test/allocated.c \ - $(srcroot)test/ALLOCM_ARENA.c $(srcroot)test/bitmap.c \ - $(srcroot)test/mremap.c $(srcroot)test/posix_memalign.c \ - $(srcroot)test/thread_arena.c $(srcroot)test/thread_tcache_enabled.c -ifeq ($(enable_experimental), 1) -CTESTS += $(srcroot)test/allocm.c $(srcroot)test/rallocm.c +C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \ + $(srcroot)test/src/btalloc_1.c $(srcroot)test/src/math.c \ + $(srcroot)test/src/mtx.c $(srcroot)test/src/mq.c \ + $(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \ + $(srcroot)test/src/thd.c $(srcroot)test/src/timer.c +ifeq (1, $(link_whole_archive)) +C_UTIL_INTEGRATION_SRCS := +else +C_UTIL_INTEGRATION_SRCS := $(srcroot)src/nstime.c $(srcroot)src/util.c endif +TESTS_UNIT := \ + $(srcroot)test/unit/a0.c \ + $(srcroot)test/unit/arena_reset.c \ + $(srcroot)test/unit/atomic.c \ + $(srcroot)test/unit/bitmap.c \ + $(srcroot)test/unit/ckh.c \ + $(srcroot)test/unit/decay.c \ + $(srcroot)test/unit/fork.c \ + $(srcroot)test/unit/hash.c \ + $(srcroot)test/unit/junk.c \ + $(srcroot)test/unit/junk_alloc.c \ + $(srcroot)test/unit/junk_free.c \ + $(srcroot)test/unit/lg_chunk.c \ + $(srcroot)test/unit/mallctl.c \ + $(srcroot)test/unit/math.c \ + $(srcroot)test/unit/mq.c \ + $(srcroot)test/unit/mtx.c \ + $(srcroot)test/unit/pack.c \ + $(srcroot)test/unit/pages.c \ + $(srcroot)test/unit/ph.c \ + $(srcroot)test/unit/prng.c \ + $(srcroot)test/unit/prof_accum.c \ + $(srcroot)test/unit/prof_active.c \ + $(srcroot)test/unit/prof_gdump.c \ + $(srcroot)test/unit/prof_idump.c \ + $(srcroot)test/unit/prof_reset.c \ + $(srcroot)test/unit/prof_thread_name.c \ + $(srcroot)test/unit/ql.c \ + $(srcroot)test/unit/qr.c \ + $(srcroot)test/unit/quarantine.c \ + $(srcroot)test/unit/rb.c \ + $(srcroot)test/unit/rtree.c \ + $(srcroot)test/unit/run_quantize.c \ + $(srcroot)test/unit/SFMT.c \ + $(srcroot)test/unit/size_classes.c \ + $(srcroot)test/unit/smoothstep.c \ + $(srcroot)test/unit/stats.c \ + $(srcroot)test/unit/ticker.c \ + $(srcroot)test/unit/nstime.c \ + $(srcroot)test/unit/tsd.c \ + $(srcroot)test/unit/util.c \ + $(srcroot)test/unit/witness.c \ + $(srcroot)test/unit/zero.c +TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ + $(srcroot)test/integration/allocated.c \ + $(srcroot)test/integration/sdallocx.c \ + $(srcroot)test/integration/mallocx.c \ + $(srcroot)test/integration/MALLOCX_ARENA.c \ + $(srcroot)test/integration/overflow.c \ + $(srcroot)test/integration/posix_memalign.c \ + $(srcroot)test/integration/rallocx.c \ + $(srcroot)test/integration/thread_arena.c \ + $(srcroot)test/integration/thread_tcache_enabled.c \ + $(srcroot)test/integration/xallocx.c \ + $(srcroot)test/integration/chunk.c +TESTS_STRESS := $(srcroot)test/stress/microbench.c +TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS) -COBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.$(O)) -CPICOBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.pic.$(O)) -CTESTOBJS := $(CTESTS:$(srcroot)%.c=$(objroot)%.$(O)) +C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O)) +C_PIC_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.pic.$(O)) +C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O)) +C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O)) +C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O)) +C_UTIL_INTEGRATION_OBJS := $(C_UTIL_INTEGRATION_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O)) +C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O)) +C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS) -.PHONY: all dist doc_html doc_man doc +TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O)) +TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O)) +TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O)) +TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS) + +.PHONY: all dist build_doc_html build_doc_man build_doc .PHONY: install_bin install_include install_lib -.PHONY: install_html install_man install_doc install +.PHONY: install_doc_html install_doc_man install_doc install .PHONY: tests check clean distclean relclean -.SECONDARY : $(CTESTOBJS) +.SECONDARY : $(TESTS_OBJS) # Default target. -all: build +all: build_lib dist: build_doc -$(srcroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl +$(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< -$(srcroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl +$(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< build_doc_html: $(DOCS_HTML) @@ -138,30 +247,45 @@ build_doc: $(DOCS) # Include generated dependency files. # ifdef CC_MM --include $(COBJS:%.$(O)=%.d) --include $(CPICOBJS:%.$(O)=%.d) --include $(CTESTOBJS:%.$(O)=%.d) +-include $(C_OBJS:%.$(O)=%.d) +-include $(C_PIC_OBJS:%.$(O)=%.d) +-include $(C_JET_OBJS:%.$(O)=%.d) +-include $(C_TESTLIB_OBJS:%.$(O)=%.d) +-include $(TESTS_OBJS:%.$(O)=%.d) endif -$(COBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c -$(CPICOBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c -$(CPICOBJS): CFLAGS += $(PIC_CFLAGS) -$(CTESTOBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c -$(CTESTOBJS): CPPFLAGS += -I$(objroot)test +$(C_OBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c +$(C_PIC_OBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c +$(C_PIC_OBJS): CFLAGS += $(PIC_CFLAGS) +$(C_JET_OBJS): $(objroot)src/%.jet.$(O): $(srcroot)src/%.c +$(C_JET_OBJS): CFLAGS += -DJEMALLOC_JET +$(C_TESTLIB_UNIT_OBJS): $(objroot)test/src/%.unit.$(O): $(srcroot)test/src/%.c +$(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST +$(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c +$(C_TESTLIB_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST +$(C_UTIL_INTEGRATION_OBJS): $(objroot)src/%.integration.$(O): $(srcroot)src/%.c +$(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%.c +$(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB +$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include +$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST +$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST +$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST +$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c +$(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include ifneq ($(IMPORTLIB),$(SO)) -$(COBJS): CPPFLAGS += -DDLLEXPORT +$(C_OBJS) $(C_JET_OBJS): CPPFLAGS += -DDLLEXPORT endif ifndef CC_MM -# Dependencies +# Dependencies. HEADER_DIRS = $(srcroot)include/jemalloc/internal \ $(objroot)include/jemalloc $(objroot)include/jemalloc/internal HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h)) -$(COBJS) $(CPICOBJS) $(CTESTOBJS): $(HEADERS) -$(CTESTOBJS): $(objroot)test/jemalloc_test.h +$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS) +$(TESTS_OBJS): $(objroot)test/include/test/jemalloc_test.h endif -$(COBJS) $(CPICOBJS) $(CTESTOBJS): %.$(O): +$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O): @mkdir -p $(@D) $(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $< ifdef CC_MM @@ -174,119 +298,180 @@ ifneq ($(SOREV),$(SO)) ln -sf $( $(objroot)$${t}.out 2>&1; \ - if test -e "$(srcroot)$${t}.exp"; then \ - diff -w -u $(srcroot)$${t}.exp \ - $(objroot)$${t}.out >/dev/null 2>&1; \ - fail=$$?; \ - if test "$${fail}" -eq "1" ; then \ - failures=`expr $${failures} + 1`; \ - echo "*** FAIL ***"; \ - else \ - echo "pass"; \ - fi; \ - else \ - echo "*** FAIL *** (.exp file is missing)"; \ - failures=`expr $${failures} + 1`; \ - fi; \ - done; \ - echo "========================================="; \ - echo "Failures: $${failures}/$${total}"' +check_unit_dir: + @mkdir -p $(objroot)test/unit +check_integration_dir: + @mkdir -p $(objroot)test/integration +stress_dir: + @mkdir -p $(objroot)test/stress +check_dir: check_unit_dir check_integration_dir + +check_unit: tests_unit check_unit_dir + $(MALLOC_CONF)="purge:ratio" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%) + $(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%) +check_integration_prof: tests_integration check_integration_dir +ifeq ($(enable_prof), 1) + $(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) + $(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) +endif +check_integration_decay: tests_integration check_integration_dir + $(MALLOC_CONF)="purge:decay,decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) + $(MALLOC_CONF)="purge:decay,decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) + $(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) +check_integration: tests_integration check_integration_dir + $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) +stress: tests_stress stress_dir + $(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%) +check: check_unit check_integration check_integration_decay check_integration_prof + +ifeq ($(enable_code_coverage), 1) +coverage_unit: check_unit + $(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS) + +coverage_integration: check_integration + $(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS) + +coverage_stress: stress + $(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress stress $(TESTS_STRESS_OBJS) + +coverage: check + $(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS) $(TESTS_UNIT_AUX_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress integration $(TESTS_STRESS_OBJS) +endif clean: - rm -f $(COBJS) - rm -f $(CPICOBJS) - rm -f $(COBJS:%.$(O)=%.d) - rm -f $(CPICOBJS:%.$(O)=%.d) - rm -f $(CTESTOBJS:%.$(O)=%$(EXE)) - rm -f $(CTESTOBJS) - rm -f $(CTESTOBJS:%.$(O)=%.d) - rm -f $(CTESTOBJS:%.$(O)=%.out) + rm -f $(C_OBJS) + rm -f $(C_PIC_OBJS) + rm -f $(C_JET_OBJS) + rm -f $(C_TESTLIB_OBJS) + rm -f $(C_OBJS:%.$(O)=%.d) + rm -f $(C_OBJS:%.$(O)=%.gcda) + rm -f $(C_OBJS:%.$(O)=%.gcno) + rm -f $(C_PIC_OBJS:%.$(O)=%.d) + rm -f $(C_PIC_OBJS:%.$(O)=%.gcda) + rm -f $(C_PIC_OBJS:%.$(O)=%.gcno) + rm -f $(C_JET_OBJS:%.$(O)=%.d) + rm -f $(C_JET_OBJS:%.$(O)=%.gcda) + rm -f $(C_JET_OBJS:%.$(O)=%.gcno) + rm -f $(C_TESTLIB_OBJS:%.$(O)=%.d) + rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcda) + rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcno) + rm -f $(TESTS_OBJS:%.$(O)=%$(EXE)) + rm -f $(TESTS_OBJS) + rm -f $(TESTS_OBJS:%.$(O)=%.d) + rm -f $(TESTS_OBJS:%.$(O)=%.gcda) + rm -f $(TESTS_OBJS:%.$(O)=%.gcno) + rm -f $(TESTS_OBJS:%.$(O)=%.out) rm -f $(DSOS) $(STATIC_LIBS) + rm -f $(objroot)*.gcov.* distclean: clean - rm -rf $(objroot)autom4te.cache + rm -f $(objroot)bin/jemalloc-config + rm -f $(objroot)bin/jemalloc.sh + rm -f $(objroot)bin/jeprof rm -f $(objroot)config.log rm -f $(objroot)config.status rm -f $(objroot)config.stamp @@ -295,7 +480,7 @@ distclean: clean relclean: distclean rm -f $(objroot)configure - rm -f $(srcroot)VERSION + rm -f $(objroot)VERSION rm -f $(DOCS_HTML) rm -f $(DOCS_MAN3) diff --git a/deps/jemalloc/README b/deps/jemalloc/README index 7661683ba..5ff24a9ef 100644 --- a/deps/jemalloc/README +++ b/deps/jemalloc/README @@ -1,10 +1,14 @@ -jemalloc is a general-purpose scalable concurrent malloc(3) implementation. -This distribution is a "portable" implementation that currently targets -FreeBSD, Linux, Apple OS X, and MinGW. jemalloc is included as the default -allocator in the FreeBSD and NetBSD operating systems, and it is used by the -Mozilla Firefox web browser on Microsoft Windows-related platforms. Depending -on your needs, one of the other divergent versions may suit your needs better -than this distribution. +jemalloc is a general purpose malloc(3) implementation that emphasizes +fragmentation avoidance and scalable concurrency support. jemalloc first came +into use as the FreeBSD libc allocator in 2005, and since then it has found its +way into numerous applications that rely on its predictable behavior. In 2010 +jemalloc development efforts broadened to include developer support features +such as heap profiling, Valgrind integration, and extensive monitoring/tuning +hooks. Modern jemalloc releases continue to be integrated back into FreeBSD, +and therefore versatility remains critical. Ongoing development efforts trend +toward making jemalloc among the best allocators for a broad range of demanding +applications, and eliminating/mitigating weaknesses that have practical +repercussions for real world applications. The COPYING file contains copyright and licensing information. @@ -13,4 +17,4 @@ jemalloc. The ChangeLog file contains a brief summary of changes for each release. -URL: http://www.canonware.com/jemalloc/ +URL: http://jemalloc.net/ diff --git a/deps/jemalloc/VERSION b/deps/jemalloc/VERSION index 5e64fc9e8..810bd6d4c 100644 --- a/deps/jemalloc/VERSION +++ b/deps/jemalloc/VERSION @@ -1 +1 @@ -3.2.0-0-g87499f6748ebe4817571e817e9f680ccb5bf54a9 +4.4.0-0-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc diff --git a/deps/jemalloc/bin/jemalloc-config.in b/deps/jemalloc/bin/jemalloc-config.in new file mode 100644 index 000000000..b016c8d33 --- /dev/null +++ b/deps/jemalloc/bin/jemalloc-config.in @@ -0,0 +1,79 @@ +#!/bin/sh + +usage() { + cat < +Options: + --help | -h : Print usage. + --version : Print jemalloc version. + --revision : Print shared library revision number. + --config : Print configure options used to build jemalloc. + --prefix : Print installation directory prefix. + --bindir : Print binary installation directory. + --datadir : Print data installation directory. + --includedir : Print include installation directory. + --libdir : Print library installation directory. + --mandir : Print manual page installation directory. + --cc : Print compiler used to build jemalloc. + --cflags : Print compiler flags used to build jemalloc. + --cppflags : Print preprocessor flags used to build jemalloc. + --ldflags : Print library flags used to build jemalloc. + --libs : Print libraries jemalloc was linked against. +EOF +} + +prefix="@prefix@" +exec_prefix="@exec_prefix@" + +case "$1" in +--help | -h) + usage + exit 0 + ;; +--version) + echo "@jemalloc_version@" + ;; +--revision) + echo "@rev@" + ;; +--config) + echo "@CONFIG@" + ;; +--prefix) + echo "@PREFIX@" + ;; +--bindir) + echo "@BINDIR@" + ;; +--datadir) + echo "@DATADIR@" + ;; +--includedir) + echo "@INCLUDEDIR@" + ;; +--libdir) + echo "@LIBDIR@" + ;; +--mandir) + echo "@MANDIR@" + ;; +--cc) + echo "@CC@" + ;; +--cflags) + echo "@CFLAGS@" + ;; +--cppflags) + echo "@CPPFLAGS@" + ;; +--ldflags) + echo "@LDFLAGS@ @EXTRA_LDFLAGS@" + ;; +--libs) + echo "@LIBS@" + ;; +*) + usage + exit 1 +esac diff --git a/deps/jemalloc/bin/pprof b/deps/jemalloc/bin/jeprof.in old mode 100755 new mode 100644 similarity index 92% rename from deps/jemalloc/bin/pprof rename to deps/jemalloc/bin/jeprof.in index 727eb4370..42087fcec --- a/deps/jemalloc/bin/pprof +++ b/deps/jemalloc/bin/jeprof.in @@ -2,11 +2,11 @@ # Copyright (c) 1998-2007, Google Inc. # All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above @@ -16,7 +16,7 @@ # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -40,28 +40,28 @@ # # Examples: # -# % tools/pprof "program" "profile" +# % tools/jeprof "program" "profile" # Enters "interactive" mode # -# % tools/pprof --text "program" "profile" +# % tools/jeprof --text "program" "profile" # Generates one line per procedure # -# % tools/pprof --gv "program" "profile" +# % tools/jeprof --gv "program" "profile" # Generates annotated call-graph and displays via "gv" # -# % tools/pprof --gv --focus=Mutex "program" "profile" +# % tools/jeprof --gv --focus=Mutex "program" "profile" # Restrict to code paths that involve an entry that matches "Mutex" # -# % tools/pprof --gv --focus=Mutex --ignore=string "program" "profile" +# % tools/jeprof --gv --focus=Mutex --ignore=string "program" "profile" # Restrict to code paths that involve an entry that matches "Mutex" # and does not match "string" # -# % tools/pprof --list=IBF_CheckDocid "program" "profile" +# % tools/jeprof --list=IBF_CheckDocid "program" "profile" # Generates disassembly listing of all routines with at least one # sample that match the --list= pattern. The listing is # annotated with the flat and cumulative sample counts at each line. # -# % tools/pprof --disasm=IBF_CheckDocid "program" "profile" +# % tools/jeprof --disasm=IBF_CheckDocid "program" "profile" # Generates disassembly listing of all routines with at least one # sample that match the --disasm= pattern. The listing is # annotated with the flat and cumulative sample counts at each PC value. @@ -72,10 +72,11 @@ use strict; use warnings; use Getopt::Long; +my $JEPROF_VERSION = "@jemalloc_version@"; my $PPROF_VERSION = "2.0"; # These are the object tools we use which can come from a -# user-specified location using --tools, from the PPROF_TOOLS +# user-specified location using --tools, from the JEPROF_TOOLS # environment variable, or from the environment. my %obj_tool_map = ( "objdump" => "objdump", @@ -94,7 +95,7 @@ my @EVINCE = ("evince"); # could also be xpdf or perhaps acroread my @KCACHEGRIND = ("kcachegrind"); my @PS2PDF = ("ps2pdf"); # These are used for dynamic profiles -my @URL_FETCHER = ("curl", "-s"); +my @URL_FETCHER = ("curl", "-s", "--fail"); # These are the web pages that servers need to support for dynamic profiles my $HEAP_PAGE = "/pprof/heap"; @@ -144,13 +145,13 @@ my $sep_address = undef; sub usage_string { return < +jeprof [options] is a space separated list of profile names. -pprof [options] +jeprof [options] is a list of profile files where each file contains the necessary symbol mappings as well as profile data (likely generated with --raw). -pprof [options] +jeprof [options] is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE Each name can be: @@ -161,9 +162,9 @@ pprof [options] $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, $CENSUSPROFILE_PAGE, or /pprof/filteredprofile. For instance: - pprof http://myserver.com:80$HEAP_PAGE + jeprof http://myserver.com:80$HEAP_PAGE If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). -pprof --symbols +jeprof --symbols Maps addresses to symbol names. In this mode, stdin should be a list of library mappings, in the same format as is found in the heap- and cpu-profile files (this loosely matches that of /proc/self/maps @@ -202,7 +203,7 @@ Output type: --pdf Generate PDF to stdout --svg Generate SVG to stdout --gif Generate GIF to stdout - --raw Generate symbolized pprof data (useful with remote fetch) + --raw Generate symbolized jeprof data (useful with remote fetch) Heap-Profile Options: --inuse_space Display in-use (mega)bytes [default] @@ -222,11 +223,14 @@ Call-graph Options: --nodefraction= Hide nodes below *total [default=.005] --edgefraction= Hide edges below *total [default=.001] --maxdegree= Max incoming/outgoing edges per node [default=8] - --focus= Focus on nodes matching - --ignore= Ignore nodes matching + --focus= Focus on backtraces with nodes matching + --thread= Show profile for thread + --ignore= Ignore backtraces with nodes matching --scale= Set GV scaling [default=0] --heapcheck Make nodes with non-0 object counts (i.e. direct leak generators) more visible + --retain= Retain only nodes that match + --exclude= Exclude all nodes that match Miscellaneous: --tools=[,...] \$PATH for object tool pathnames @@ -235,34 +239,34 @@ Miscellaneous: --version Version information Environment Variables: - PPROF_TMPDIR Profiles directory. Defaults to \$HOME/pprof - PPROF_TOOLS Prefix for object tools pathnames + JEPROF_TMPDIR Profiles directory. Defaults to \$HOME/jeprof + JEPROF_TOOLS Prefix for object tools pathnames Examples: -pprof /bin/ls ls.prof +jeprof /bin/ls ls.prof Enters "interactive" mode -pprof --text /bin/ls ls.prof +jeprof --text /bin/ls ls.prof Outputs one line per procedure -pprof --web /bin/ls ls.prof +jeprof --web /bin/ls ls.prof Displays annotated call-graph in web browser -pprof --gv /bin/ls ls.prof +jeprof --gv /bin/ls ls.prof Displays annotated call-graph via 'gv' -pprof --gv --focus=Mutex /bin/ls ls.prof +jeprof --gv --focus=Mutex /bin/ls ls.prof Restricts to code paths including a .*Mutex.* entry -pprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof +jeprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof Code paths including Mutex but not string -pprof --list=getdir /bin/ls ls.prof +jeprof --list=getdir /bin/ls ls.prof (Per-line) annotated source listing for getdir() -pprof --disasm=getdir /bin/ls ls.prof +jeprof --disasm=getdir /bin/ls ls.prof (Per-PC) annotated disassembly for getdir() -pprof http://localhost:1234/ +jeprof http://localhost:1234/ Enters "interactive" mode -pprof --text localhost:1234 +jeprof --text localhost:1234 Outputs one line per procedure for localhost:1234 -pprof --raw localhost:1234 > ./local.raw -pprof --text ./local.raw +jeprof --raw localhost:1234 > ./local.raw +jeprof --text ./local.raw Fetches a remote profile for later analysis and then analyzes it in text mode. EOF @@ -270,7 +274,8 @@ EOF sub version_string { return < \$main::opt_edgefraction, "maxdegree=i" => \$main::opt_maxdegree, "focus=s" => \$main::opt_focus, + "thread=s" => \$main::opt_thread, "ignore=s" => \$main::opt_ignore, "scale=i" => \$main::opt_scale, "heapcheck" => \$main::opt_heapcheck, + "retain=s" => \$main::opt_retain, + "exclude=s" => \$main::opt_exclude, "inuse_space!" => \$main::opt_inuse_space, "inuse_objects!" => \$main::opt_inuse_objects, "alloc_space!" => \$main::opt_alloc_space, @@ -562,6 +573,86 @@ sub Init() { } } +sub FilterAndPrint { + my ($profile, $symbols, $libs, $thread) = @_; + + # Get total data in profile + my $total = TotalProfile($profile); + + # Remove uniniteresting stack items + $profile = RemoveUninterestingFrames($symbols, $profile); + + # Focus? + if ($main::opt_focus ne '') { + $profile = FocusProfile($symbols, $profile, $main::opt_focus); + } + + # Ignore? + if ($main::opt_ignore ne '') { + $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); + } + + my $calls = ExtractCalls($symbols, $profile); + + # Reduce profiles to required output granularity, and also clean + # each stack trace so a given entry exists at most once. + my $reduced = ReduceProfile($symbols, $profile); + + # Get derived profiles + my $flat = FlatProfile($reduced); + my $cumulative = CumulativeProfile($reduced); + + # Print + if (!$main::opt_interactive) { + if ($main::opt_disasm) { + PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm); + } elsif ($main::opt_list) { + PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); + } elsif ($main::opt_text) { + # Make sure the output is empty when have nothing to report + # (only matters when --heapcheck is given but we must be + # compatible with old branches that did not pass --heapcheck always): + if ($total != 0) { + printf("Total%s: %s %s\n", + (defined($thread) ? " (t$thread)" : ""), + Unparse($total), Units()); + } + PrintText($symbols, $flat, $cumulative, -1); + } elsif ($main::opt_raw) { + PrintSymbolizedProfile($symbols, $profile, $main::prog); + } elsif ($main::opt_callgrind) { + PrintCallgrind($calls); + } else { + if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { + if ($main::opt_gv) { + RunGV(TempName($main::next_tmpfile, "ps"), ""); + } elsif ($main::opt_evince) { + RunEvince(TempName($main::next_tmpfile, "pdf"), ""); + } elsif ($main::opt_web) { + my $tmp = TempName($main::next_tmpfile, "svg"); + RunWeb($tmp); + # The command we run might hand the file name off + # to an already running browser instance and then exit. + # Normally, we'd remove $tmp on exit (right now), + # but fork a child to remove $tmp a little later, so that the + # browser has time to load it first. + delete $main::tempnames{$tmp}; + if (fork() == 0) { + sleep 5; + unlink($tmp); + exit(0); + } + } + } else { + cleanup(); + exit(1); + } + } + } else { + InteractiveMode($profile, $symbols, $libs, $total); + } +} + sub Main() { Init(); $main::collected_profile = undef; @@ -605,9 +696,6 @@ sub Main() { $symbol_map = MergeSymbols($symbol_map, $base->{symbols}); } - # Get total data in profile - my $total = TotalProfile($profile); - # Collect symbols my $symbols; if ($main::use_symbolized_profile) { @@ -622,75 +710,17 @@ sub Main() { $symbols = ExtractSymbols($libs, $pcs); } - # Remove uniniteresting stack items - $profile = RemoveUninterestingFrames($symbols, $profile); - - # Focus? - if ($main::opt_focus ne '') { - $profile = FocusProfile($symbols, $profile, $main::opt_focus); + if (!defined($main::opt_thread)) { + FilterAndPrint($profile, $symbols, $libs); } - - # Ignore? - if ($main::opt_ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); - } - - my $calls = ExtractCalls($symbols, $profile); - - # Reduce profiles to required output granularity, and also clean - # each stack trace so a given entry exists at most once. - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - # Print - if (!$main::opt_interactive) { - if ($main::opt_disasm) { - PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm); - } elsif ($main::opt_list) { - PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); - } elsif ($main::opt_text) { - # Make sure the output is empty when have nothing to report - # (only matters when --heapcheck is given but we must be - # compatible with old branches that did not pass --heapcheck always): - if ($total != 0) { - printf("Total: %s %s\n", Unparse($total), Units()); - } - PrintText($symbols, $flat, $cumulative, -1); - } elsif ($main::opt_raw) { - PrintSymbolizedProfile($symbols, $profile, $main::prog); - } elsif ($main::opt_callgrind) { - PrintCallgrind($calls); - } else { - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), ""); - } elsif ($main::opt_evince) { - RunEvince(TempName($main::next_tmpfile, "pdf"), ""); - } elsif ($main::opt_web) { - my $tmp = TempName($main::next_tmpfile, "svg"); - RunWeb($tmp); - # The command we run might hand the file name off - # to an already running browser instance and then exit. - # Normally, we'd remove $tmp on exit (right now), - # but fork a child to remove $tmp a little later, so that the - # browser has time to load it first. - delete $main::tempnames{$tmp}; - if (fork() == 0) { - sleep 5; - unlink($tmp); - exit(0); - } - } - } else { - cleanup(); - exit(1); + if (defined($data->{threads})) { + foreach my $thread (sort { $a <=> $b } keys(%{$data->{threads}})) { + if (defined($main::opt_thread) && + ($main::opt_thread eq '*' || $main::opt_thread == $thread)) { + my $thread_profile = $data->{threads}{$thread}; + FilterAndPrint($thread_profile, $symbols, $libs, $thread); } } - } else { - InteractiveMode($profile, $symbols, $libs, $total); } cleanup(); @@ -780,14 +810,14 @@ sub InteractiveMode { $| = 1; # Make output unbuffered for interactive mode my ($orig_profile, $symbols, $libs, $total) = @_; - print STDERR "Welcome to pprof! For help, type 'help'.\n"; + print STDERR "Welcome to jeprof! For help, type 'help'.\n"; # Use ReadLine if it's installed and input comes from a console. if ( -t STDIN && !ReadlineMightFail() && defined(eval {require Term::ReadLine}) ) { - my $term = new Term::ReadLine 'pprof'; - while ( defined ($_ = $term->readline('(pprof) '))) { + my $term = new Term::ReadLine 'jeprof'; + while ( defined ($_ = $term->readline('(jeprof) '))) { $term->addhistory($_) if /\S/; if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { last; # exit when we get an interactive command to quit @@ -795,7 +825,7 @@ sub InteractiveMode { } } else { # don't have readline while (1) { - print STDERR "(pprof) "; + print STDERR "(jeprof) "; $_ = ; last if ! defined $_ ; s/\r//g; # turn windows-looking lines into unix-looking lines @@ -988,7 +1018,7 @@ sub ProcessProfile { sub InteractiveHelpMessage { print STDERR <