diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 91b34ebe0..9452c4204 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,20 +1,24 @@ --- name: Bug report -about: Create a report to help us improve -title: '' +about: Help us improve KeyDB by reporting a bug +title: '[BUG]' labels: '' assignees: '' --- **Describe the bug** -A clear and concise description of what the bug is. -** Log Files ** -These should be KeyDB logs, not syslogs or logs from your container manager. If you are reporting a crash there will be a line in your log stating: -"=== KEYDB BUG REPORT START: Cut & paste starting from here ===" +A short description of the bug. -Please copy everything after this line. +**To reproduce** -**To Reproduce** -Do you know how to reproduce this? If so please provide repro steps. +Steps to reproduce the behavior and/or a minimal code sample. + +**Expected behavior** + +A description of what you expected to happen. + +**Additional information** + +Any additional information that is relevant to the problem. diff --git a/.github/ISSUE_TEMPLATE/crash_report.md b/.github/ISSUE_TEMPLATE/crash_report.md new file mode 100644 index 000000000..c608ccdc1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crash_report.md @@ -0,0 +1,20 @@ +--- +name: Crash report +about: Submit a crash report +title: '[CRASH]' +labels: '' +assignees: '' + +--- + +**Crash report** + +Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context. + +``` +``` + +**Aditional information** + +1. OS distribution and version +2. Steps to reproduce (if any) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d6..22dfc5631 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,20 +1,24 @@ --- name: Feature request -about: Suggest an idea for this project -title: '' +about: Suggest a feature for KeyDB +title: '[NEW]' labels: '' assignees: '' --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +**The problem/use-case that the feature addresses** -**Describe the solution you'd like** -A clear and concise description of what you want to happen. +A description of the problem that the feature will solve, or the use-case with which the feature will be used. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +**Description of the feature** -**Additional context** -Add any other context or screenshots about the feature request here. +A description of what you want to happen. + +**Alternatives you've considered** + +Any alternative solutions or features you've considered, including references to existing open and closed feature requests in this repository. + +**Additional information** + +Any additional information that is relevant to the feature request. diff --git a/.github/ISSUE_TEMPLATE/other_stuff.md b/.github/ISSUE_TEMPLATE/other_stuff.md new file mode 100644 index 000000000..e82cf0024 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/other_stuff.md @@ -0,0 +1,8 @@ +--- +name: Other +about: Can't find the right issue type? Use this one! +title: '' +labels: '' +assignees: '' + +--- diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..6acde0d06 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,21 @@ +--- +name: Question +about: Ask the Redis developers +title: '[QUESTION]' +labels: '' +assignees: '' + +--- + +Please keep in mind that this issue tracker should be used for reporting bugs or proposing improvements to the Redis server. + +Generally, questions about using Redis should be directed to the [community](https://redis.io/community): + +* [the mailing list](https://groups.google.com/forum/#!forum/redis-db) +* [the `redis` tag at StackOverflow](http://stackoverflow.com/questions/tagged/redis) +* [/r/redis subreddit](http://www.reddit.com/r/redis) +* [the irc channel #redis](http://webchat.freenode.net/?channels=redis) on freenode + +It is also possible that your question was already asked here, so please do a quick issues search before submitting. Lastly, if your question is about one of Redis' [clients](https://redis.io/clients), you may to contact your client's developers for help. + +That said, please feel free to replace all this with your question :) \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0327be059..e7f22f48e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get update sudo apt-get -y remove libzstd || true sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev - make BUILD_TLS=yes -j2 + make BUILD_TLS=yes -j2 REDIS_CFLAGS='-Werror' - name: gen-cert run: ./utils/gen-test-certs.sh - name: test-tls diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 6e4f88ef3..ee9ac1bbf 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -17,19 +17,19 @@ jobs: steps: - uses: actions/checkout@v2 - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make + run: make REDIS_CFLAGS='-Werror -DREDIS_TEST' - name: test run: | - sudo apt-get install tcl8.5 - ./runtest --accurate --verbose + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs - name: module api test run: ./runtest-moduleapi --verbose - name: sentinel tests run: ./runtest-sentinel - name: cluster tests run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all test-ubuntu-libc-malloc: runs-on: ubuntu-latest @@ -38,13 +38,11 @@ jobs: steps: - uses: actions/checkout@v2 - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make MALLOC=libc + run: make MALLOC=libc - name: test run: | - sudo apt-get install tcl8.5 - ./runtest --accurate --verbose + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs - name: module api test run: ./runtest-moduleapi --verbose - name: sentinel tests @@ -52,7 +50,26 @@ jobs: - name: cluster tests run: ./runtest-cluster - test: + test-ubuntu-no-malloc-usable-size: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-ubuntu-32bit: runs-on: ubuntu-latest if: github.repository == 'redis/redis' timeout-minutes: 14400 @@ -60,33 +77,67 @@ jobs: - uses: actions/checkout@v2 - name: make run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make BUILD_TLS=yes -j2 - - name: "test (tls)" - run: | - sudo apt-get install tcl8.5 tcl-tls - ./utils/gen-test-certs.sh - ./runtest --accurate --verbose --tls + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST' - name: test - run: ./runtest --accurate --verbose - - name: module api test (tls) - run: ./runtest-moduleapi --verbose --tls + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: | + make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time + ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all - test-ubuntu-arm: - runs-on: [self-hosted, linux, arm] + test-ubuntu-tls: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 steps: - uses: actions/checkout@v2 - name: make run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make -j4 + make BUILD_TLS=yes - name: test run: | - sudo apt-get -y install tcl8.5 - ./runtest --clients 2 --verbose - - name: module tests + sudo apt-get install tcl8.6 tcl-tls + ./utils/gen-test-certs.sh + ./runtest --accurate --verbose --tls --dump-logs + ./runtest --accurate --verbose --dump-logs + - name: module api test run: | - ./runtest-moduleapi + ./runtest-moduleapi --verbose --tls + ./runtest-moduleapi --verbose + - name: sentinel tests + run: | + ./runtest-sentinel --tls + ./runtest-sentinel + - name: cluster tests + run: | + ./runtest-cluster --tls + ./runtest-cluster + + test-ubuntu-io-threads: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + make + - name: test + run: | + sudo apt-get install tcl8.6 tcl-tls + ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs + - name: cluster tests + run: | + ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes test-valgrind: runs-on: ubuntu-latest @@ -95,16 +146,34 @@ jobs: steps: - uses: actions/checkout@v2 - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make valgrind + run: make valgrind REDIS_CFLAGS='-Werror -DREDIS_TEST' - name: test run: | sudo apt-get update - sudo apt-get install tcl8.5 valgrind -y - ./runtest --valgrind --verbose --clients 1 + sudo apt-get install tcl8.6 valgrind -y + ./runtest --valgrind --verbose --clients 1 --dump-logs - name: module api test - run: ./runtest-moduleapi --valgrind --verbose --clients 1 + run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 + - name: unittest + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all + if grep -q 0x err.txt; then cat err.txt; exit 1; fi + + test-valgrind-no-malloc-usable-size: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE" + - name: test + run: | + sudo apt-get update + sudo apt-get install tcl8.6 valgrind -y + ./runtest --valgrind --verbose --clients 1 --dump-logs + - name: module api test + run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 test-centos7-jemalloc: runs-on: ubuntu-latest @@ -115,13 +184,12 @@ jobs: - uses: actions/checkout@v2 - name: make run: | - yum -y install centos-release-scl - yum -y install devtoolset-7 - scl enable devtoolset-7 "make" + yum -y install gcc make + make - name: test run: | - yum -y install tcl - ./runtest --accurate --verbose + yum -y install which tcl + ./runtest --accurate --verbose --dump-logs - name: module api test run: ./runtest-moduleapi --verbose - name: sentinel tests @@ -145,8 +213,8 @@ jobs: run: | yum -y install tcl tcltls ./utils/gen-test-certs.sh - ./runtest --accurate --verbose --tls - ./runtest --accurate --verbose + ./runtest --accurate --verbose --tls --dump-logs + ./runtest --accurate --verbose --dump-logs - name: module api test run: | ./runtest-moduleapi --verbose --tls @@ -170,7 +238,7 @@ jobs: run: make - name: test run: | - ./runtest --accurate --verbose --no-latency + ./runtest --accurate --verbose --no-latency --dump-logs - name: module api test run: ./runtest-moduleapi --verbose - name: sentinel tests @@ -178,3 +246,63 @@ jobs: - name: cluster tests run: ./runtest-cluster + test-freebsd: + runs-on: macos-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: test + uses: vmactions/freebsd-vm@v0.1.2 + with: + usesh: true + sync: rsync + prepare: pkg install -y bash gmake lang/tcl86 + run: > + gmake && + ./runtest --accurate --verbose --no-latency --dump-logs && + MAKE=gmake ./runtest-moduleapi --verbose && + ./runtest-sentinel && + ./runtest-cluster + + test-alpine-jemalloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: alpine:latest + steps: + - uses: actions/checkout@v2 + - name: make + run: | + apk add build-base + make REDIS_CFLAGS='-Werror' + - name: test + run: | + apk add tcl procps + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-alpine-libc-malloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: alpine:latest + steps: + - uses: actions/checkout@v2 + - name: make + run: | + apk add build-base + make REDIS_CFLAGS='-Werror' USE_JEMALLOC=no CFLAGS=-DUSE_MALLOC_USABLE_SIZE + - name: test + run: | + apk add tcl procps + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster diff --git a/.gitignore b/.gitignore index f85cfbc81..f6b4ed0b6 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ Makefile.dep .ccls .ccls-cache/* compile_commands.json +redis.code-workspace diff --git a/00-RELEASENOTES b/00-RELEASENOTES index bff270e77..8a1405e41 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,4 +1,4 @@ -Redis 6.0 release notes +Redis 6.2 release notes ======================= -------------------------------------------------------------------------------- @@ -12,25 +12,246 @@ SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- ================================================================================ -Redis 6.0.10 Released Tue Jan 12 16:20:20 IST 2021 +Redis 6.2.2 Released Mon April 19 19:00:00 IST 2021 ================================================================================ -Upgrade urgency MODERATE: several bugs with moderate impact are fixed, -Here is a comprehensive list of changes in this release compared to 6.0.9. +Upgrade urgency: HIGH, if you're using ACL and pub/sub, CONFIG REWRITE, or +suffering from performance regression. see below. + +Bug fixes for regressions in previous releases of Redis 6.2: +* Fix BGSAVE, AOFRW, and replication slowdown due to child reporting CoW (#8645) +* Fix short busy loop when timer event is about to fire (#8764) +* Fix default user, overwritten and reset users losing pubsub channel permissions (#8723) +* Fix config rewrite with an empty `save` config resulsing in default `save` values (#8719) +* Fix not starting on alpine/libmusl without IPv6 (#8655) +* Fix issues with propagation and MULTI/EXEC in modules (#8617) + Several issues around nested calls and thread safe contexts + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* ACL Pub/Sub channels permission handling for save/load scenario (#8794) +* Fix early rejection of PUBLISH inside MULTI-EXEC transaction (#8534) +* Fix missing SLOWLOG records for blocked commands (#8632) +* Allow RESET command during busy scripts (#8629) +* Fix some error replies were not counted on stats (#8659) + +Bug fixes: +* Add a timeout mechanism for replicas stuck in fullsync (#8762) +* Process HELLO command even if the default user has no permissions (#8633) +* Client issuing a long running script and using a pipeline, got disconnected (#8715) +* Fix script kill to work also on scripts that use `pcall` (#8661) +* Fix list-compress-depth may compress more node than required (#8311) +* Fix redis-cli handling of rediss:// URL scheme (#8705) +* Cluster: Skip unnecessary check which may prevent failure detection (#8585) +* Cluster: Fix hang manual failover when replica just started (#8651) +* Sentinel: Fix info-refresh time field before sentinel get first response (#8567) +* Sentinel: Fix possible crash on failed connection attempt (#8627) +* Systemd: Send the readiness notification when a replica is ready to accept connections (#8409) Command behavior changes: -* SWAPDB invalidates WATCHed keys (#8239) -* SORT command behaves differently when used on a writable replica (#8283) -* EXISTS should not alter LRU (#8016) - In Redis 5.0 and 6.0 it would have touched the LRU/LFU of the key. -* OBJECT should not reveal logically expired keys (#8016) - Will now behave the same TYPE or any other non-DEBUG command. -* GEORADIUS[BYMEMBER] can fail with -OOM if Redis is over the memory limit (#8107) +* ZADD: fix wrong reply when INCR used with GT/LT which blocked the update (#8717) + It was responding with the incremented value rather than nil +* XAUTOCLAIM: fix response to return the next available id as the cursor (#8725) + Previous behavior was retuning the last one which was already scanned +* XAUTOCLAIM: fix JUSTID to prevent incrementing delivery_count (#8724) + +New config options: +* Add cluster-allow-replica-migration config option (#5285) +* Add replica-announced config option (#8653) +* Add support for plaintext clients in TLS cluster (#8587) +* Add support for reading encrypted keyfiles (#8644) + +Improvements: +* Fix performance regression in BRPOP on Redis 6.0 (#8689) +* Avoid adding slowlog entries for config with sensitive data (#8584) +* Improve redis-cli non-binary safe string handling (#8566) +* Optimize CLUSTER SLOTS reply (#8541) +* Handle remaining fsync errors (#8419) + +Info fields and introspection changes: +* Strip % sign from current_fork_perc info field (#8628) +* Fix RSS memory info on FreeBSD (#8620) +* Fix client_recent_max_input/output_buffer in 'INFO CLIENTS' when all clients drop (#8588) +* Fix invalid master_link_down_since_seconds in info replication (#8785) + +Platform and deployment-related changes: +* Fix FreeBSD <12.x builds (#8603) + +Modules: +* Add macros for RedisModule_log logging levels (#4246) +* Add RedisModule_GetAbsExpire / RedisModule_SetAbsExpire (#8564) +* Add a module type for key space notification (#8759) +* Set module eviction context flag only in masters (#8631) +* Fix unusable RedisModule_IsAOFClient API (#8596) +* Fix missing EXEC on modules propagation after failed EVAL execution (#8654) +* Fix edge-case when a module client is unblocked (#8618) + +================================================================================ +Redis 6.2.1 Released Mon Mar 1 17:51:36 IST 2021 +================================================================================ + +Upgrade urgency: LOW. + +Here is a comprehensive list of changes in this release compared to 6.2.0, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +Bug fixes: +* Fix sanitize-dump-payload for stream with deleted records (#8568) +* Prevent client-query-buffer-limit config from being set to lower than 1mb (#8557) + +Improvements: +* Make port, tls-port and bind config options modifiable at runtime (#8510) + +Platform and deployment-related changes: +* Fix compilation error on non-glibc systems if jemalloc is not used (#8533) +* Improved memory consumption and memory usage tracking on FreeBSD (#8545) +* Fix compilation on ARM64 MacOS with jemalloc (#8458) + +Modules: +* New Module API for getting user name of a client (#8508) +* Optimize RM_Call by utilizing a shared reusable client (#8516) +* Fix crash running CLIENT INFO via RM_Call (#8560) + +================================================================================ +Redis 6.2.0 GA Released Tue Feb 22 14:00:00 IST 2021 +================================================================================ + +Upgrade urgency: SECURITY if you use 32bit build of redis (see bellow), MODERATE +if you used earlier versions of Redis 6.2, LOW otherwise. + +Integer overflow on 32-bit systems (CVE-2021-21309): +Redis 4.0 or newer uses a configurable limit for the maximum supported bulk +input size. By default, it is 512MB which is a safe value for all platforms. +If the limit is significantly increased, receiving a large request from a client +may trigger several integer overflow scenarios, which would result with buffer +overflow and heap corruption. + +Here is a comprehensive list of changes in this release compared to 6.2 RC3, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +Bug fixes: +* Avoid 32-bit overflows when proto-max-bulk-len is set high (#8522) +* Fix broken protocol in client tracking tracking-redir-broken message (#8456) +* Avoid unsafe field name characters in INFO commandstats, errorstats, modules (#8492) +* XINFO able to access expired keys during CLIENT PAUSE WRITE (#8436) +* Fix allowed length for REPLCONF ip-address, needed due to Sentinel's support for hostnames (#8517) +* Fix broken protocol in redis-benchmark when used with -a or --dbnum (#8486) +* XADD counts deleted records too when considering switching to a new listpack (#8390) + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fixes in GEOSEARCH bybox (accuracy and mismatch between width and height) (#8445) +* Fix risk of OOM panic in HRANDFIELD, ZRANDMEMBER commands with huge negative count (#8429) +* Fix duplicate replicas issue in Sentinel, needed due to hostname support (#8481) +* Fix Sentinel configuration rewrite, an improvement of #8271 (#8480) + +Command behavior changes: +* SRANDMEMBER uses RESP3 array type instead of set type (#8504) +* EXPIRE, EXPIREAT, SETEX, GETEX: Return error when provided expire time overflows (#8287) Other behavior changes: +* Remove ACL subcommand validation if fully added command exists. (#8483) + +Improvements: +* Optimize sorting in GEORADIUS / GEOSEARCH with COUNT (#8326) +* Optimize HRANDFIELD and ZRANDMEMBER case 4 when ziplist encoded (#8444) +* Optimize in-place replacement of elements in HSET, HINCRBY, LSET (#8493) +* Remove redundant list to store pubsub patterns (#8472) +* Add --insecure option to command line tools (#8416) + +Info fields and introspection changes: +* Add INFO fields to track progress of BGSAVE, AOFRW, replication (#8414) + +Modules: +* RM_ZsetRem: Delete key if empty, the bug could leave empty zset keys (#8453) +* RM_HashSet: Add COUNT_ALL flag and set errno (#8446) + +================================================================================ +Redis 6.2 RC3 Released Tue Feb 1 14:00:00 IST 2021 +================================================================================ + +Upgrade urgency LOW: This is the third Release Candidate of Redis 6.2. + +Here is a comprehensive list of changes in this release compared to 6.2 RC2, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add HRANDFIELD and ZRANDMEMBER commands (#8297) +* Add FAILOVER command (#8315) +* Add GETEX, GETDEL commands (#8327) +* Add PXAT/EXAT arguments to SET command (#8327) +* Add SYNC arg to FLUSHALL and FLUSHDB, and ASYNC/SYNC arg to SCRIPT FLUSH (#8258) + +Sentinel: +* Add hostname support to Sentinel (#8282) +* Prevent file descriptors from leaking into Sentinel scripts (#8242) +* Fix config file line order dependency and config rewrite sequence (#8271) + +New configuration options: +* Add set-proc-title config option to disable changes to the process title (#3623) +* Add proc-title-template option to control what's shown in the process title (#8397) +* Add lazyfree-lazy-user-flush config option to control FLUSHALL, FLUSHDB and SCRIPT FLUSH (#8258) + +Bug fixes: +* AOF: recover from last write error by turning on/off appendonly config (#8030) +* Exit on fsync error when the AOF fsync policy is 'always' (#8347) +* Avoid assertions (on older kernels) when testing arm64 CoW bug (#8405) +* CONFIG REWRITE should honor umask settings (#8371) +* Fix firstkey,lastkey,step in COMMAND command for some commands (#8367) + +Special considerations: +* Fix misleading description of the save configuration directive (#8337) + +Improvements: +* A way to get RDB file via replication without excessive replication buffers (#8303) +* Optimize performance of clusterGenNodesDescription for large clusters (#8182) + +Info fields and introspection changes: +* SLOWLOG and LATENCY monitor include unblocking time of blocked commands (#7491) + +Modules: +* Add modules API for streams (#8288) +* Add event for fork child birth and termination (#8289) +* Add RM_BlockedClientMeasureTime* etc, to track background processing in commandstats (#7491) +* Fix bug in v6.2, wrong value passed to the new unlink callback (#8381) +* Fix bug in v6.2, modules blocked on keys unblock on commands like LPUSH (#8356) + +================================================================================ +Redis 6.2 RC2 Released Tue Jan 12 16:17:20 IST 2021 +================================================================================ + +Upgrade urgency LOW: This is the second Release Candidate of Redis 6.2. + +IMPORTANT: If you're running Redis on ARM64 or a big-endian system, upgrade may +have significant implications. Please be sure to read the notes below. + +Here is a comprehensive list of changes in this release compared to 6.2 RC1, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add the REV, BYLEX and BYSCORE arguments to ZRANGE, and the ZRANGESTORE command (#7844) +* Add the XAUTOCLAIM command (#7973) +* Add the MINID trimming strategy and the LIMIT argument to XADD and XTRIM (#8169) +* Add the ANY argument to GEOSEARCH and GEORADIUS (#8259) +* Add the CH, NX, XX arguments to GEOADD (#8227) +* Add the COUNT argument to LPOP and RPOP (#8179) +* Add the WRITE argument to CLIENT PAUSE for pausing write commands exclusively (#8170) +* Change the proto-ver argument of HELLO to optional (#7377) +* Add the CLIENT TRACKINGINFO subcommand (#7309) + +Command behavior changes: +* CLIENT TRACKING yields an error when given overlapping BCAST prefixes (#8176) +* SWAPDB invalidates WATCHed keys (#8239) +* SORT command behaves differently when used on a writable replica (#8283) + +Other behavior changes: +* Avoid propagating MULTI/EXEC for read-only transactions (#8216) +* Remove the read-only flag from TIME, ECHO, ROLE, LASTSAVE (#8216) +* Fix the command flags of PFDEBUG (#8222) +* Tracking clients will no longer receive unnecessary key invalidation messages after FLUSHDB (#8039) * Sentinel: Fix missing updates to the config file after SENTINEL SET command (#8229) -* CONFIG REWRITE is atomic and safer, but requires write access to the config file's folder (#7824, #8051) - This change was already present in 6.0.9, but was missing from the release notes. Bug fixes with compatibility implications (bugs introduced in Redis 6.0): * Fix RDB CRC64 checksum on big-endian systems (#8270) @@ -40,17 +261,148 @@ Bug fixes with compatibility implications (bugs introduced in Redis 6.0): If your scripts use redis.setresp() or return a map (new in Redis 6.0), please consider the implications. +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Resolve rare assertions in active defragmentation while loading (#8284, #8281) + Bug fixes: +* Fix the selection of a random element from large hash tables (#8133) * Fix an issue where a forked process deletes the parent's pidfile (#8231) * Fix crashes when enabling io-threads-do-reads (#8230) * Fix a crash in redis-cli after executing cluster backup (#8267) +* Fix redis-benchmark to use an IP address for the first cluster node (#8154) +* Fix saving of strings larger than 2GB into RDB files (#8306) + +Additional improvements: +* Improve replication handshake time (#8214) +* Release client tracking table memory asynchronously in cases where the DB is also freed asynchronously (#8039) +* Avoid wasteful transient memory allocation in certain cases (#8286, #5954) +* Handle binary string values by the 'requirepass' and 'masterauth' configs (#8200) + +Platform and deployment-related changes: +* Install redis-check-rdb and redis-check-aof as symlinks to redis-server (#5745) +* Add a check for an ARM64 Linux kernel bug (#8224) + Due to the potential severity of this issue, Redis will refuse to run on + affected platforms by default. + +Info fields and introspection changes: +* Add the errorstats section to the INFO command (#8217) +* Add the failed_calls and rejected_calls fields INFO's commandstats section (#8217) +* Report child copy-on-write metrics continuously (#8264) + +Module API changes: +* Add the RedisModule_SendChildCOWInfo API (#8264) +* Add the may-replicate command flag (#8170) + +================================================================================ +Redis 6.2 RC1 Released Mon Dec 14 11:50:00 IST 2020 +================================================================================ + +Upgrade urgency LOW: This is the first Release Candidate of Redis 6.2. + +Introduction to the Redis 6.2 release +===================================== + +This release is the first significant Redis release managed by the core team +under the new project governance model. + +Redis 6.2 includes many new commands and improvements, but no big features. It +mainly makes Redis more complete and addresses issues that have been requested +by many users frequently or for a long time. + +Many of these changes were not eligible for 6.0.x for several reasons: + +1. They are not backward compatible, which is always the case with new or + extended commands (that cannot be replicated to an older replica). +2. They require a longer release-candidate test cycle. + + +Here is a comprehensive list of changes in this release compared to 6.0.9, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add SMISMEMBER command that checks multiple members (#7615) +* Add ZMSCORE command that returns an array of scores (#7593) +* Add LMOVE and BLMOVE commands that pop and push arbitrarily (#6929) +* Add RESET command that resets client connection state (#7982) +* Add COPY command that copies keys (#7953) +* Add ZDIFF and ZDIFFSTORE commands (#7961) +* Add ZINTER and ZUNION commands (#7794) +* Add GEOSEARCH/GEOSEARCHSTORE commands for bounding box spatial queries (#8094) +* Add GET parameter to SET command, for more powerful GETSET (#7852) +* Add exclusive range query to XPENDING (#8130) +* Add exclusive range query to X[REV]RANGE (#8072) +* Add GT and LT options to ZADD for conditional score updates (#7818) +* Add CLIENT INFO and CLIENT LIST for specific ids (#8113) +* Add IDLE argument to XPENDING command (#7972) +* Add local address to CLIENT LIST, and a CLIENT KILL filter. (#7913) +* Add NOMKSTREAM option to XADD command (#7910) +* Add command introspection to Sentinel (#7940) +* Add SENTINEL MYID subcommand (#7858) + +New features: +* Dump payload sanitization: prevent corrupt payload causing crashes (#7807) + Has flags to enable full O(N) validation (disabled by default). +* ACL patterns for Pub/Sub channels (#7993) +* Support ACL for Sentinel mode (#7888) +* Support getting configuration from both stdin and file at the same time (#7893) + Lets you avoid storing secrets on the disk. + +New features in CLI tools: +* redis-cli RESP3 push support (#7609) +* redis-cli cluster import support source and target that require auth (#7994) +* redis-cli URIs able to provide user name in addition to password (#8048) +* redis-cli/redis-benchmark allow specifying the prefered ciphers/ciphersuites (#8005) +* redis-cli add -e option to exit with code when command execution fails (#8136) + +Command behavior changes: +* EXISTS should not alter LRU (#8016) + In Redis 5.0 and 6.0 it would have touched the LRU/LFU of the key. +* OBJECT should not reveal logically expired keys (#8016) + Will now behave the same TYPE or any other non-DEBUG command. +* Improve db id range check for SELECT and MOVE (#8085) + Changes the error message text on a wrong db index. +* Modify AUTH / HELLO error message (#7648) + Changes the error message text when the user isn't found or is disabled. +* BITOPS length limited to proto_max_bulk_len rather than 512MB (#8096) + The limit is now configurable like in SETRANGE, and APPEND. +* GEORADIUS[BYMEMBER] can fail with -OOM if Redis is over the memory limit (#8107) + +Other behavior changes: +* Optionally (default) fail to start if requested bind address is not available (#7936) + If you rely on Redis starting successfully even if one of the bind addresses + is not available, you'll need to tune the new config. +* Limit the main db dictionaries expansion to prevent key eviction (#7954) + In the past big dictionary rehashing could result in massive data eviction. + Now this rehashing is delayed (up to a limit), which can result in performance + loss due to hash collisions. +* CONFIG REWRITE is atomic and safer, but requires write access to the config file's folder (#7824, #8051) + This change was already present in 6.0.9, but was missing from the release + notes. +* A new incremental eviction mechanism that reduces latency on eviction spikes (#7653) + In pathological cases this can cause memory to grow uncontrolled and may require + specific tuning. +* Not resetting "save" config when Redis is started with command line arguments. (#7092) + In case you provide command line arguments without "save" and count on it + being disabled, Now the defaults "save" config will kick in. +* Update memory metrics for INFO during loading (#7690) +* When "supervised" config is enabled, it takes precedence over "daemonize". (#8036) +* Assertion and panic, print crash log without generating SIGSEGV (#7585) +* Added crash log report on SIGABRT, instead of silently exiting (#8004) +* Disable THP (Transparent Huge Pages) if enabled (#7381) + If you deliberately enabled it, you'll need to config Redis to keep it. + +Bug fixes: * Handle output buffer limits for module blocked clients (#8141) - Could result in a module sending reply to a blocked client to go beyond the limit. + Could result in a module sending reply to a blocked client to go beyond the + limit. * Fix setproctitle related crashes. (#8150, #8088) - Caused various crashes on startup, mainly on Apple M1 chips or under instrumentation. + Caused various crashes on startup, mainly on Apple M1 chips or under + instrumentation. +* A module doing RM_Call could cause replicas to get nested MULTI (#8097). * Backup/restore cluster mode keys to slots map for repl-diskless-load=swapdb (#8108) - In cluster mode with repl-diskless-load, when loading failed, slot map wouldn't - have been restored. + In cluster mode with repl-diskless-load, when loading failed, slot map + wouldn't have been restored. * Fix oom-score-adj-values range, and bug when used in config file (#8046) Enabling setting this in the config file in a line after enabling it, would have been buggy. @@ -62,3219 +414,119 @@ Bug fixes: Output of ACL GETUSER is now more similar to the one provided by ACL SETUSER. * Fix bug with module GIL being released prematurely (#8061) Could in theory (and rarely) cause multi-threaded modules to corrupt memory. +* Fix cluster redirect for module command with no firstkey. (#7539) * Reduce effect of client tracking causing feedback loop in key eviction (#8100) -* Fix cluster access to unaligned memory (SIGBUS on old ARM) (#7958) -* Fix saving of strings larger than 2GB into RDB files (#8306) +* Kill disk-based fork child when all replicas drop and 'save' is not enabled (#7819) +* Rewritten commands (modified for propagation) are logged as their original command (#8006) +* Fix cluster access to unaligned memory (SIGBUS on old ARM) #7958 +* If diskless repl child is killed, make sure to reap the child pid (#7742) +* Broadcast a PONG message when slot's migration is over, may reduce MOVED responses (#7571) -Additional improvements: -* Avoid wasteful transient memory allocation in certain cases (#8286, #5954) +Other improvements: +* TLS Support in redis-benchmark (#7959) +* Accelerate diskless master connections, and general re-connections (#6271) +* Run active defrag while blocked / loading (#7726) +* Performance and memory reporting improvement - sds take control of its internal fragmentation (#7875) +* Speedup cluster failover. (#7948) Platform / toolchain support related improvements: +* Optionally (not by default) use H/W Monotonic clock for faster time sampling (#7644) +* Remove the requirements for C11 and _Atomic supporting compiler (#7707) + This would allow to more easily build and use Redis on older systems and + compilers again. * Fix crash log registers output on ARM. (#8020) -* Add a check for an ARM64 Linux kernel bug (#8224) - Due to the potential severity of this issue, Redis will print log warning on startup. * Raspberry build fix. (#8095) +* Setting process title support for Haiku. (#8060) +* DragonFlyBSD RSS memory sampling support. (#8023) New configuration options: +* Enable configuring OpenSSL using the standard openssl.cnf (#8143) * oom-score-adj-values config can now take absolute values (besides relative ones) (#8046) +* TLS: Add different client cert support. (#8076) +* Note that a few other changes listed above added their config options. + +Info fields and introspection changes: +* Add INFO fields to track diskless and disk-based replication progress (#7981) +* Add INFO field for main thread cpu time, and scrape system time. (#8132) +* Add total_forks to INFO STATS (#8155) +* Add maxclients and cluster_connections to INFO CLIENTS (#7979) +* Add tracking bcast flag and client redirection in client list (#7995) +* Fixed INFO client_recent_max_input_buffer includes argv array (#8065, see #7874) +* Note that a few other changes listed above added their info fields. + +Module API changes: +* Add CTX_FLAGS_DENY_BLOCKING as a unified the way to know if blocking is allowed (#8025) +* Add data type callbacks for lazy free effort, and unlink (#7912) +* Add data type callback for COPY command (#8112) +* Add callbacks for defrag support. (#8149) +* Add module event for repl-diskless-load swapdb (#8153) Module related fixes: * Moved RMAPI_FUNC_SUPPORTED so that it's usable (#8037) * Improve timer accuracy (#7987) * Allow '\0' inside of result of RM_CreateStringPrintf (#6260) -================================================================================ -Redis 6.0.9 Released Mon Oct 26 10:37:47 IST 2020 -================================================================================ - -Upgrade urgency: SECURITY if you use an affected platform (see below). - Otherwise the upgrade urgency is MODERATE. - -This release fixes a potential heap overflow when using a heap allocator other -than jemalloc or glibc's malloc. See: -https://github.com/redis/redis/pull/7963 - -Other fixes in this release: - -New: -* Memory reporting of clients argv (#7874) -* Add redis-cli control on raw format line delimiter (#7841) -* Add redis-cli support for rediss:// -u prefix (#7900) -* Get rss size support for NetBSD and DragonFlyBSD - -Behavior changes: -* WATCH no longer ignores keys which have expired for MULTI/EXEC (#7920) -* Correct OBJECT ENCODING response for stream type (#7797) -* Allow blocked XREAD on a cluster replica (#7881) -* TLS: Do not require CA config if not used (#7862) - -Bug fixes: -* INFO report real peak memory (before eviction) (#7894) -* Allow requirepass config to clear the password (#7899) -* Fix config rewrite file handling to make it really atomic (#7824) -* Fix excessive categories being displayed from ACLs (#7889) -* Add fsync in replica when full RDB payload was received (#7839) -* Don't write replies to socket when output buffer limit reached (#7202) -* Fix redis-check-rdb support for modules aux data (#7826) -* Other smaller bug fixes - -Modules API: -* Add APIs for version and compatibility checks (#7865) -* Add RM_GetClientCertificate (#7866) -* Add RM_GetDetachedThreadSafeContext (#7886) -* Add RM_GetCommandKeys (#7884) -* Add Swapdb Module Event (#7804) -* RM_GetContextFlags provides indication of being in a fork child (#7783) -* RM_GetContextFlags document missing flags: MULTI_DIRTY, IS_CHILD (#7821) -* Expose real client on connection events (#7867) -* Minor improvements to module blocked on keys (#7903) - -Full list of commits: - -Yossi Gottlieb in commit ce0d74d8f: - Fix wrong zmalloc_size() assumption. (#7963) - 1 file changed, 3 deletions(-) - -Oran Agra in commit d3ef26822: - Attempt to fix sporadic test failures due to wait_for_log_messages (#7955) - 1 file changed, 2 insertions(+) - -David CARLIER in commit 76993a0d4: - cpu affinity: DragonFlyBSD support (#7956) - 2 files changed, 9 insertions(+), 2 deletions(-) - -Zach Fewtrell in commit b23cdc14a: - fix invalid 'failover' identifier in cluster slave selection test (#7942) - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 99a4cb401: - Update rdb_last_bgsave_time_sec in INFO on diskless replication (#7917) - 1 file changed, 11 insertions(+), 14 deletions(-) - -Wen Hui in commit 258287c35: - do not add save parameter during config rewrite in sentinel mode (#7945) - 1 file changed, 6 insertions(+) - -Qu Chen in commit 6134279e2: - WATCH no longer ignores keys which have expired for MULTI/EXEC. (#7920) - 2 files changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit d15ec67c6: - improve verbose logging on failed test. print log file lines (#7938) - 1 file changed, 4 insertions(+) - -Yossi Gottlieb in commit 8a2e6d24f: - Add a --no-latency tests flag. (#7939) - 5 files changed, 23 insertions(+), 9 deletions(-) - -filipe oliveira in commit 0a1737dc5: - Fixed bug concerning redis-benchmark non clustered benchmark forcing always the same hash tag {tag} (#7931) - 1 file changed, 31 insertions(+), 24 deletions(-) - -Oran Agra in commit 6d9b3df71: - fix 32bit build warnings (#7926) - 2 files changed, 3 insertions(+), 3 deletions(-) - -Wen Hui in commit ed6f7a55e: - fix double fclose in aofrewrite (#7919) - 1 file changed, 6 insertions(+), 5 deletions(-) - -Oran Agra in commit 331d73c92: - INFO report peak memory before eviction (#7894) - 1 file changed, 11 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit e88e13528: - Fix tests failure on busybox systems. (#7916) - 2 files changed, 2 insertions(+), 2 deletions(-) - -Oran Agra in commit b7f53738e: - Allow requirepass config to clear the password (#7899) - 1 file changed, 18 insertions(+), 8 deletions(-) - -Wang Yuan in commit 2ecb28b68: - Remove temporary aof and rdb files in a background thread (#7905) - 2 files changed, 3 insertions(+), 3 deletions(-) - -guybe7 in commit 7bc605e6b: - Minor improvements to module blocked on keys (#7903) - 3 files changed, 15 insertions(+), 9 deletions(-) - -Andreas Lind in commit 1b484608d: - Support redis-cli -u rediss://... (#7900) - 1 file changed, 9 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 95095d680: - Modules: fix RM_GetCommandKeys API. (#7901) - 3 files changed, 4 insertions(+), 7 deletions(-) - -Meir Shpilraien (Spielrein) in commit cd3ae2f2c: - Add Module API for version and compatibility checks (#7865) - 9 files changed, 180 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit 1d723f734: - Module API: Add RM_GetClientCertificate(). (#7866) - 6 files changed, 88 insertions(+) - -Yossi Gottlieb in commit d72172752: - Modules: Add RM_GetDetachedThreadSafeContext(). (#7886) - 4 files changed, 52 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit e4f9aff19: - Modules: add RM_GetCommandKeys(). - 6 files changed, 238 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 6682b913e: - Introduce getKeysResult for getKeysFromCommand. - 7 files changed, 170 insertions(+), 121 deletions(-) - -Madelyn Olson in commit 9db65919c: - Fixed excessive categories being displayed from acls (#7889) - 2 files changed, 29 insertions(+), 2 deletions(-) - -Oran Agra in commit f34c50cf6: - Add some additional signal info to the crash log (#7891) - 1 file changed, 4 insertions(+), 1 deletion(-) - -Oran Agra in commit 300bb4701: - Allow blocked XREAD on a cluster replica (#7881) - 3 files changed, 43 insertions(+) - -Oran Agra in commit bc5cf0f1a: - memory reporting of clients argv (#7874) - 5 files changed, 55 insertions(+), 5 deletions(-) - -DvirDukhan in commit 13d2e6a57: - redis-cli add control on raw format line delimiter (#7841) - 1 file changed, 8 insertions(+), 6 deletions(-) - -Oran Agra in commit d54e25620: - Include internal sds fragmentation in MEMORY reporting (#7864) - 2 files changed, 7 insertions(+), 7 deletions(-) - -Oran Agra in commit ac2c2b74e: - Fix crash in script timeout during AOF loading (#7870) - 2 files changed, 47 insertions(+), 4 deletions(-) - -Rafi Einstein in commit 00d2082e7: - Makefile: enable program suffixes via PROG_SUFFIX (#7868) - 2 files changed, 10 insertions(+), 6 deletions(-) - -nitaicaro in commit d2c2c26e7: - Fixed Tracking test “The other connection is able to get invalidations” (#7871) - 1 file changed, 3 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 2c172556f: - Modules: expose real client on conn events. - 1 file changed, 11 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 2972d0c1f: - Module API: Fail ineffective auth calls. - 1 file changed, 5 insertions(+) - -Yossi Gottlieb in commit aeb2a3b6a: - TLS: Do not require CA config if not used. (#7862) - 1 file changed, 5 insertions(+), 3 deletions(-) - -Oran Agra in commit d8e64aeb8: - warning: comparison between signed and unsigned integer in 32bit build (#7838) - 1 file changed, 2 insertions(+), 2 deletions(-) - -David CARLIER in commit 151209982: - Add support for Haiku OS (#7435) - 3 files changed, 16 insertions(+) - -Gavrie Philipson in commit b1d3e169f: - Fix typo in module API docs (#7861) - 1 file changed, 2 insertions(+), 2 deletions(-) - -David CARLIER in commit 08e3b8d13: - getting rss size implementation for netbsd (#7293) - 1 file changed, 20 insertions(+) - -Oran Agra in commit 0377a889b: - Fix new obuf-limits tests to work with TLS (#7848) - 2 files changed, 29 insertions(+), 13 deletions(-) - -caozb in commit a057ad9b1: - ignore slaveof no one in redis.conf (#7842) - 1 file changed, 10 insertions(+), 1 deletion(-) - -Wang Yuan in commit 87ecee645: - Don't support Gopher if enable io threads to read queries (#7851) - 2 files changed, 8 insertions(+), 5 deletions(-) - -Wang Yuan in commit b92902236: - Set 'loading' and 'shutdown_asap' to volatile sig_atomic_t type (#7845) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Uri Shachar in commit ee0875a02: - Fix config rewrite file handling to make it really atomic (#7824) - 1 file changed, 49 insertions(+), 47 deletions(-) - -WuYunlong in commit d577519e1: - Add fsync to readSyncBulkPayload(). (#7839) - 1 file changed, 11 insertions(+) - -Wen Hui in commit 104e0ea3e: - rdb.c: handle fclose error case differently to avoid double fclose (#7307) - 1 file changed, 7 insertions(+), 6 deletions(-) - -Wang Yuan in commit 0eb015ac6: - Don't write replies if close the client ASAP (#7202) - 7 files changed, 144 insertions(+), 2 deletions(-) - -Guy Korland in commit 08a03e32c: - Fix RedisModule_HashGet examples (#6697) - 1 file changed, 4 insertions(+), 4 deletions(-) - -Oran Agra in commit 09551645d: - fix recently broken TLS build error, and add coverage for CI (#7833) - 2 files changed, 4 insertions(+), 3 deletions(-) - -David CARLIER in commit c545ba5d0: - Further NetBSD update and build fixes. (#7831) - 3 files changed, 72 insertions(+), 3 deletions(-) - -WuYunlong in commit ec9050053: - Fix redundancy use of semicolon in do-while macros in ziplist.c. (#7832) - 1 file changed, 3 insertions(+), 3 deletions(-) - -yixiang in commit 27a4d1314: - Fix connGetSocketError usage (#7811) - 2 files changed, 6 insertions(+), 4 deletions(-) - -Oran Agra in commit 30795dcae: - RM_GetContextFlags - document missing flags (#7821) - 1 file changed, 6 insertions(+) - -Yossi Gottlieb in commit 14a12849f: - Fix occasional hangs on replication reconnection. (#7830) - 2 files changed, 14 insertions(+), 3 deletions(-) - -Ariel Shtul in commit d5a1b06dc: - Fix redis-check-rdb support for modules aux data (#7826) - 3 files changed, 21 insertions(+), 1 deletion(-) - -Wen Hui in commit 39f793693: - refactor rewriteStreamObject code for adding missing streamIteratorStop call (#7829) - 1 file changed, 36 insertions(+), 18 deletions(-) - -WuYunlong in commit faad29bfb: - Make IO threads killable so that they can be canceled at any time. - 1 file changed, 1 insertion(+) - -WuYunlong in commit b3f1b5830: - Make main thread killable so that it can be canceled at any time. Refine comment of makeThreadKillable(). - 3 files changed, 11 insertions(+), 4 deletions(-) - -Oran Agra in commit 0f43d1f55: - RM_GetContextFlags provides indication that we're in a fork child (#7783) - 8 files changed, 28 insertions(+), 18 deletions(-) - -Wen Hui in commit a55ea9cdf: - Add Swapdb Module Event (#7804) - 5 files changed, 52 insertions(+) - -Daniel Dai in commit 1d8f72bef: - fix make warnings in debug.c MacOS (#7805) - 2 files changed, 3 insertions(+), 2 deletions(-) - -David CARLIER in commit 556953d93: - debug.c: NetBSD build warning fix. (#7810) - 1 file changed, 4 insertions(+), 3 deletions(-) - -Wang Yuan in commit d02435b66: - Remove tmp rdb file in background thread (#7762) - 6 files changed, 82 insertions(+), 8 deletions(-) - -Oran Agra in commit 1bd7bfdc0: - Add printf attribute and fix warnings and a minor bug (#7803) - 2 files changed, 12 insertions(+), 4 deletions(-) - -WuYunlong in commit d25147b4c: - bio: doFastMemoryTest should try to kill io threads as well. - 3 files changed, 19 insertions(+) - -WuYunlong in commit 4489ba081: - bio: fix doFastMemoryTest. - 4 files changed, 25 insertions(+), 3 deletions(-) - -Wen Hui in commit cf85def67: - correct OBJECT ENCODING response for stream type (#7797) - 1 file changed, 1 insertion(+) - -WuYunlong in commit cf5bcf892: - Clarify help text of tcl scripts. (#7798) - 1 file changed, 1 insertion(+) - -Mykhailo Pylyp in commit f72665c65: - Recalculate hardcoded variables from $::instances_count in sentinel tests (#7561) - 3 files changed, 15 insertions(+), 13 deletions(-) - -Oran Agra in commit c67b19e7a: - Fix failing valgrind installation in github actions (#7792) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 92763fd2a: - fix broken PEXPIREAT test (#7791) - 1 file changed, 10 insertions(+), 6 deletions(-) - -Wang Yuan in commit f5b4c0ccb: - Remove dead global variable 'lru_clock' (#7782) - 1 file changed, 1 deletion(-) - -Oran Agra in commit 82d431fd6: - Squash merging 125 typo/grammar/comment/doc PRs (#7773) - 80 files changed, 436 insertions(+), 416 deletions(-) - -================================================================================ -Redis 6.0.8 Released Wed Sep 09 23:34:17 IDT 2020 -================================================================================ - -Upgrade urgency HIGH: Anyone who's using Redis 6.0.7 with Sentinel or -CONFIG REWRITE command is affected and should upgrade ASAP, see #7760. - -Bug fixes: - -* CONFIG REWRITE after setting oom-score-adj-values either via CONFIG SET or - loading it from a config file, will generate a corrupt config file that will - cause Redis to fail to start -* Fix issue with redis-cli --pipe on MacOS -* Fix RESP3 response for HKEYS/HVALS on non-existing key -* Various small bug fixes - -New features / Changes: - -* Remove THP warning when set to madvise -* Allow EXEC with read commands on readonly replica in cluster -* Add masters/replicas options to redis-cli --cluster call command - -Module API: - -* Add RedisModule_ThreadSafeContextTryLock - -Full list of commits: - -Oran Agra in commit cdabf696a: - Fix RESP3 response for HKEYS/HVALS on non-existing key - 1 file changed, 3 insertions(+), 1 deletion(-) - -Oran Agra in commit ec633c716: - Fix leak in new blockedclient module API test - 1 file changed, 3 insertions(+) - -Yossi Gottlieb in commit 6bac07c5c: - Tests: fix oom-score-adj false positives. (#7772) - 1 file changed, 1 insertion(+), 1 deletion(-) - -杨博东 in commit 6043dc614: - Tests: Add aclfile load and save tests (#7765) - 2 files changed, 41 insertions(+) - -Roi Lipman in commit c0b5f9bf0: - RM_ThreadSafeContextTryLock a non-blocking method for acquiring GIL (#7738) - 7 files changed, 122 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 5780a1599: - Tests: validate CONFIG REWRITE for all params. (#7764) - 6 files changed, 43 insertions(+), 6 deletions(-) - -Oran Agra in commit e3c14b25d: - Change THP warning to use madvise rather than never (#7771) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit 28929917b: - Documents RM_Call's fmt (#5448) - 1 file changed, 25 insertions(+) - -Jan-Erik Rediger in commit 9146402c2: - Check that THP is not set to always (madvise is ok) (#4001) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Yossi Gottlieb in commit d05089429: - Tests: clean up stale .cli files. (#7768) - 1 file changed, 2 insertions(+) - -Eran Liberty in commit 8861c1bae: - Allow exec with read commands on readonly replica in cluster (#7766) - 3 files changed, 59 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit 2cf2ff2f6: - Fix CONFIG REWRITE of oom-score-adj-values. (#7761) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Oran Agra in commit 1386c80f7: - handle cur_test for nested tests - 1 file changed, 3 insertions(+) - -Oran Agra in commit c7d4945f0: - Add daily CI for MacOS (#7759) - 1 file changed, 18 insertions(+) - -bodong.ybd in commit 32548264c: - Tests: Some fixes for macOS - 3 files changed, 26 insertions(+), 11 deletions(-) - -Oran Agra in commit 1e17f9812: - Fix cluster consistency-check test (#7754) - 1 file changed, 55 insertions(+), 29 deletions(-) - -Yossi Gottlieb in commit f4ecdf86a: - Tests: fix unmonitored servers. (#7756) - 1 file changed, 5 insertions(+) - -Oran Agra in commit 9f020050d: - fix broken cluster/sentinel tests by recent commit (#7752) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit fdbabb496: - Improve valgrind support for cluster tests (#7725) - 3 files changed, 83 insertions(+), 23 deletions(-) - -Oran Agra in commit 35a6a0bbc: - test infra - add durable mode to work around test suite crashing - 3 files changed, 35 insertions(+), 3 deletions(-) - -Oran Agra in commit e3136b13f: - test infra - wait_done_loading - 2 files changed, 16 insertions(+), 36 deletions(-) - -Oran Agra in commit 83c75dbd9: - test infra - flushall between tests in external mode - 1 file changed, 1 insertion(+) - -Oran Agra in commit 265f5d3cf: - test infra - improve test skipping ability - 3 files changed, 91 insertions(+), 36 deletions(-) - -Oran Agra in commit fcd3a9908: - test infra - reduce disk space usage - 3 files changed, 33 insertions(+), 11 deletions(-) - -Oran Agra in commit b6ea4699f: - test infra - write test name to logfile - 3 files changed, 35 insertions(+) - -Yossi Gottlieb in commit 4a4b07fc6: - redis-cli: fix writeConn() buffer handling. (#7749) - 1 file changed, 37 insertions(+), 6 deletions(-) - -Oran Agra in commit f2d08de2e: - Print server startup messages after daemonization (#7743) - 1 file changed, 4 insertions(+), 4 deletions(-) - -Thandayuthapani in commit 77541d555: - Add masters/replicas options to redis-cli --cluster call command (#6491) - 1 file changed, 13 insertions(+), 2 deletions(-) - -Oran Agra in commit 91d13a854: - fix README about BUILD_WITH_SYSTEMD usage (#7739) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Yossi Gottlieb in commit 88d03d965: - Fix double-make issue with make && make install. (#7734) - 1 file changed, 2 insertions(+) - -================================================================================ -Redis 6.0.7 Released Fri Aug 28 11:05:09 IDT 2020 -================================================================================ - -Upgrade urgency MODERATE: several bugs with moderate impact are fixed, -Specifically the first two listed below which cause protocol errors for clients. - -Bug fixes: - -* CONFIG SET could hung the client when arrives during RDB/ROF loading (When - processed after another command that was also rejected with -LOADING error) -* LPOS command when RANK is greater than matches responded wiht broken protocol - (negative multi-bulk count) -* UNLINK / Lazyfree for stream type key would have never do async freeing -* PERSIST should invalidate WATCH (Like EXPIRE does) -* EXEC with only read commands could have be rejected when OOM -* TLS: relax verification on CONFIG SET (Don't error if some configs are set - and tls isn't enabled) -* TLS: support cluster/replication without tls-port -* Systemd startup after network is online -* Redis-benchmark improvements -* Various small bug fixes - -New features: - -* Add oom-score-adj configuration option to control Linux OOM killer -* Show IO threads statistics and status in INFO output -* Add optional tls verification mode (see tls-auth-clients) - -Module API: - -* Add RedisModule_HoldString -* Add loaded keyspace event -* Fix RedisModuleEvent_LoadingProgress -* Fix RedisModuleEvent_MasterLinkChange hook missing on successful psync -* Fix missing RM_CLIENTINFO_FLAG_SSL -* Refactor redismodule.h for use with -fno-common / extern - -Full list of commits: - -Oran Agra in commit c26394e4f: - Reduce the probability of failure when start redis in runtest-cluster #7554 (#7635) - 1 file changed, 23 insertions(+), 5 deletions(-) - -Leoš Literák in commit 745d5e802: - Update README.md with instructions how to build with systemd support (#7730) - 1 file changed, 5 insertions(+) - -Yossi Gottlieb in commit 03f1d208a: - Fix oom-score-adj on older distros. (#7724) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 941174d9c: - Backport Lua 5.2.2 stack overflow fix. (#7733) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wang Yuan in commit c897dba14: - Fix wrong format specifiers of 'sdscatfmt' for the INFO command (#7706) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wen Hui in commit 5e3fab5e7: - fix make warnings (#7692) - 1 file changed, 4 insertions(+), 3 deletions(-) - -Nathan Scott in commit a2b09c13f: - Annotate module API functions in redismodule.h for use with -fno-common (#6900) - 1 file changed, 265 insertions(+), 241 deletions(-) - -Yossi Gottlieb in commit bf244273f: - Add oom-score-adj configuration option to control Linux OOM killer. (#1690) - 8 files changed, 306 insertions(+), 1 deletion(-) - -Meir Shpilraien (Spielrein) in commit b5a6ab98f: - see #7544, added RedisModule_HoldString api. (#7577) - 4 files changed, 83 insertions(+), 8 deletions(-) - -ShooterIT in commit ff04cf62b: - [Redis-benchmark] Remove zrem test, add zpopmin test - 1 file changed, 5 insertions(+), 5 deletions(-) - -ShooterIT in commit 0f3260f31: - [Redis-benchmark] Support zset type - 1 file changed, 16 insertions(+) - -Arun Ranganathan in commit 45d0b94fc: - Show threading configuration in INFO output (#7446) - 3 files changed, 46 insertions(+), 14 deletions(-) - -Meir Shpilraien (Spielrein) in commit a22f61e12: - This PR introduces a new loaded keyspace event (#7536) - 8 files changed, 135 insertions(+), 4 deletions(-) - -Oran Agra in commit 1c9ca1030: - Fix rejectCommand trims newline in shared error objects, hung clients (#7714) - 4 files changed, 42 insertions(+), 23 deletions(-) - -valentinogeron in commit 217471795: - EXEC with only read commands should not be rejected when OOM (#7696) - 2 files changed, 51 insertions(+), 8 deletions(-) - -Itamar Haber in commit 6e6c47d16: - Expands lazyfree's effort estimate to include Streams (#5794) - 1 file changed, 24 insertions(+) - -Yossi Gottlieb in commit da6813623: - Add language servers stuff, test/tls to gitignore. (#7698) - 1 file changed, 4 insertions(+) - -Valentino Geron in commit de7fb126e: - Assert that setDeferredAggregateLen isn't called with negative value - 1 file changed, 1 insertion(+) - -Valentino Geron in commit 6cf27f25f: - Fix LPOS command when RANK is greater than matches - 2 files changed, 9 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 9bba54ace: - Tests: fix redis-cli with remote hosts. (#7693) - 3 files changed, 5 insertions(+), 5 deletions(-) - -huangzhw in commit 0fec2cb81: - RedisModuleEvent_LoadingProgress always at 100% progress (#7685) - 1 file changed, 2 insertions(+), 2 deletions(-) - -guybe7 in commit 931e19aa6: - Modules: Invalidate saved_oparray after use (#7688) - 1 file changed, 2 insertions(+) - -杨博东 in commit 6f2065570: - Fix flock cluster config may cause failure to restart after kill -9 (#7674) - 4 files changed, 31 insertions(+), 7 deletions(-) - -Raghav Muddur in commit 200149a2a: - Update clusterMsgDataPublish to clusterMsgModule (#7682) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Madelyn Olson in commit 72daa1b4e: - Fixed hset error since it's shared with hmset (#7678) - 1 file changed, 1 insertion(+), 1 deletion(-) - -guybe7 in commit 3bf9ac994: - PERSIST should signalModifiedKey (Like EXPIRE does) (#7671) - 1 file changed, 1 insertion(+) - -Oran Agra in commit b37501684: - OOM Crash log include size of allocation attempt. (#7670) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Wen Hui in commit 2136cb68f: - [module] using predefined REDISMODULE_NO_EXPIRE in RM_GetExpire (#7669) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Oran Agra in commit f56aee4bc: - Trim trailing spaces in error replies coming from rejectCommand (#7668) - 1 file changed, 5 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 012d7506a: - Module API: fix missing RM_CLIENTINFO_FLAG_SSL. (#7666) - 6 files changed, 82 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit a0adbc857: - TLS: relax verification on CONFIG SET. (#7665) - 2 files changed, 24 insertions(+), 7 deletions(-) - -Madelyn Olson in commit 2ef29715b: - Fixed timer warning (#5953) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wagner Francisco Mezaroba in commit b76f171f5: - allow --pattern to be used along with --bigkeys (#3586) - 1 file changed, 9 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit cc7b57765: - redis-benchmark: fix wrong random key for hset (#4895) - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 479c1ba77: - CLIENT_MASTER should ignore server.proto_max_bulk_len - 1 file changed, 2 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit f61ce8a52: - config: proto-max-bulk-len must be 1mb or greater - 2 files changed, 2 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit 0350f597a: - using proto-max-bulk-len in checkStringLength for SETRANGE and APPEND - 1 file changed, 2 insertions(+), 2 deletions(-) - -YoongHM in commit eea63548d: - Start redis after network is online (#7639) - 1 file changed, 2 insertions(+) - -Yossi Gottlieb in commit aef6d74fb: - Run daily workflow on main repo only (no forks). (#7646) - 1 file changed, 7 insertions(+) - -WuYunlong in commit 917b4d241: - see #7250, fix signature of RedisModule_DeauthenticateAndCloseClient (#7645) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wang Yuan in commit efab7fd54: - Print error info if failed opening config file (#6943) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Wen Hui in commit 8c4468bcf: - fix memory leak in ACLLoadFromFile error handling (#7623) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 89724e1d2: - redis-cli --cluster-yes - negate force flag for clarity - 1 file changed, 9 insertions(+), 9 deletions(-) - -Frank Meier in commit c813739af: - reintroduce REDISCLI_CLUSTER_YES env variable in redis-cli - 1 file changed, 6 insertions(+) - -Frank Meier in commit 7e3b86c18: - add force option to 'create-cluster create' script call (#7612) - 1 file changed, 6 insertions(+), 2 deletions(-) - -Oran Agra in commit 3f7fa4312: - fix new rdb test failing on timing issues (#7604) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 417976d7a: - Fix test-centos7-tls daily job. (#7598) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Oran Agra in commit c41818c51: - module hook for master link up missing on successful psync (#7584) - 2 files changed, 22 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 6ef3fc185: - CI: Add daily CentOS 7.x jobs. (#7582) - 1 file changed, 50 insertions(+), 4 deletions(-) - -WuYunlong in commit 002c37482: - Fix running single test 14-consistency-check.tcl (#7587) - 1 file changed, 1 insertion(+) - -Yossi Gottlieb in commit 66cbbb6ad: - Clarify RM_BlockClient() error condition. (#6093) - 1 file changed, 9 insertions(+) - -namtsui in commit 22aba2207: - Avoid an out-of-bounds read in the redis-sentinel (#7443) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Wen Hui in commit af08887dc: - Add SignalModifiedKey hook in XGROUP CREATE with MKSTREAM option (#7562) - 1 file changed, 1 insertion(+) - -Wen Hui in commit a5e0a64b0: - fix leak in error handling of debug populate command (#7062) - 1 file changed, 3 insertions(+), 4 deletions(-) - -Yossi Gottlieb in commit cbfdfa231: - Fix TLS cluster tests. (#7578) - 1 file changed, 4 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 6d5376d30: - TLS: Propagate and handle SSL_new() failures. (#7576) - 4 files changed, 48 insertions(+), 6 deletions(-) - -Oran Agra in commit a662cd577: - Fix failing tests due to issues with wait_for_log_message (#7572) - 3 files changed, 38 insertions(+), 34 deletions(-) - -Jiayuan Chen in commit 2786a4b5e: - Add optional tls verification (#7502) - 6 files changed, 40 insertions(+), 5 deletions(-) - -Oran Agra in commit 3ef3d3612: - Daily github action: run cluster and sentinel tests with tls (#7575) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit f20f63322: - TLS: support cluster/replication without tls-port. - 2 files changed, 5 insertions(+), 4 deletions(-) - -grishaf in commit 3c9ae059d: - Fix prepareForShutdown function declaration (#7566) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 3f4803af9: - Stabilize bgsave test that sometimes fails with valgrind (#7559) - 1 file changed, 20 insertions(+), 2 deletions(-) - -Madelyn Olson in commit 1a3c51a1f: - Properly reset errno for rdbLoad (#7542) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 92d80b13a: - testsuite may leave servers alive on error (#7549) - 1 file changed, 3 insertions(+) - -Yossi Gottlieb in commit 245582ba7: - Tests: drop TCL 8.6 dependency. (#7548) - 1 file changed, 27 insertions(+), 22 deletions(-) - -Oran Agra in commit f20e1ba2d: - Fixes to release scripts (#7547) - 2 files changed, 2 insertions(+), 2 deletions(-) - -Remi Collet in commit 60ff56993: - Fix deprecated tail syntax in tests (#7543) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wen Hui in commit 34e8541b9: - Add missing calls to raxStop (#7532) - 4 files changed, 63 insertions(+), 19 deletions(-) - -Wen Hui in commit 2f7bc5435: - add missing caching command in client help (#7399) - 1 file changed, 1 insertion(+) - -zhaozhao.zz in commit c15be9ffe: - replication: need handle -NOPERM error after send ping (#7538) - 1 file changed, 1 insertion(+) - -Scott Brenner in commit 1b29152c3: - GitHub Actions workflows - use latest version of actions/checkout (#7534) - 2 files changed, 10 insertions(+), 10 deletions(-) - -================================================================================ -Redis 6.0.6 Released Mon Jul 20 09:31:30 IDT 2020 -================================================================================ - -Upgrade urgency MODERATE: several bugs with moderate impact are fixed here. - -The most important issues are listed here: - -* Fix crash when enabling CLIENT TRACKING with prefix -* EXEC always fails with EXECABORT and multi-state is cleared -* RESTORE ABSTTL won't store expired keys into the db -* redis-cli better handling of non-pritable key names -* TLS: Ignore client cert when tls-auth-clients off -* Tracking: fix invalidation message on flush -* Notify systemd on Sentinel startup -* Fix crash on a misuse of STRALGO -* Few fixes in module API -* Fix a few rare leaks (STRALGO error misuse, Sentinel) -* Fix a possible invalid access in defrag of scripts (unlikely to cause real harm) - -New features: - -* LPOS command to search in a list -* Use user+pass for MIGRATE in redis-cli and redis-benchmark in cluster mode -* redis-cli support TLS for --pipe, --rdb and --replica options -* TLS: Session caching configuration support - -And this is the full list of commits: - -Itamar Haber in commit 50548cafc: - Adds SHA256SUM to redis-stable tarball upload - 1 file changed, 1 insertion(+) - -yoav-steinberg in commit 3a4c6684f: - Support passing stack allocated module strings to moduleCreateArgvFromUserFormat (#7528) - 1 file changed, 4 insertions(+), 1 deletion(-) - -Luke Palmer in commit 2fd0b2bd6: - Send null for invalidate on flush (#7469) - 1 file changed, 14 insertions(+), 10 deletions(-) - -dmurnane in commit c3c81e1a8: - Notify systemd on sentinel startup (#7168) - 1 file changed, 4 insertions(+) - -Developer-Ecosystem-Engineering in commit e2770f29b: - Add registers dump support for Apple silicon (#7453) - 1 file changed, 54 insertions(+), 2 deletions(-) - -Wen Hui in commit b068eae97: - correct error msg for num connections reaching maxclients in cluster mode (#7444) - 1 file changed, 2 insertions(+), 2 deletions(-) - -WuYunlong in commit e6169ae5c: - Fix command help for unexpected options (#7476) - 6 files changed, 20 insertions(+), 3 deletions(-) - -WuYunlong in commit abf08fc02: - Refactor RM_KeyType() by using macro. (#7486) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 11b83076a: - diskless master disconnect replicas when rdb child failed (#7518) - 1 file changed, 6 insertions(+), 5 deletions(-) - -Oran Agra in commit 8f27f2f7d: - redis-cli tests, fix valgrind timing issue (#7519) - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 180b588e8: - Fix out of update help info in tcl tests. (#7516) - 1 file changed, 2 deletions(-) - -Qu Chen in commit 417c60bdc: - Replica always reports master's config epoch in CLUSTER NODES output. (#7235) - 1 file changed, 5 insertions(+), 1 deletion(-) - -Oran Agra in commit 72a242419: - RESTORE ABSTTL skip expired keys - leak (#7511) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 2ca45239f: - fix recently added time sensitive tests failing with valgrind (#7512) - 2 files changed, 12 insertions(+), 6 deletions(-) - -Oran Agra in commit 123dc8b21: - runtest --stop pause stops before terminating the redis server (#7513) - 2 files changed, 8 insertions(+), 2 deletions(-) - -Oran Agra in commit a6added45: - update release scripts for new hosts, and CI to run more tests (#7480) - 5 files changed, 68 insertions(+), 26 deletions(-) - -jimgreen2013 in commit cf4869f9e: - fix description about ziplist, the code is ok (#6318) - 1 file changed, 2 insertions(+), 2 deletions(-) - -马永泽 in commit d548f219b: - fix benchmark in cluster mode fails to authenticate (#7488) - 1 file changed, 56 insertions(+), 40 deletions(-) - -Abhishek Soni in commit e58eb7b89: - fix: typo in CI job name (#7466) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Jiayuan Chen in commit 6def10a2b: - Fix typo in deps README (#7500) - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 8af61afef: - Add missing latency-monitor tcl test to test_helper.tcl. (#6782) - 1 file changed, 1 insertion(+) - -Yossi Gottlieb in commit a419f400e: - TLS: Session caching configuration support. (#7420) - 6 files changed, 56 insertions(+), 16 deletions(-) - -Yossi Gottlieb in commit 2e4bb2667: - TLS: Ignore client cert when tls-auth-clients off. (#7457) - 1 file changed, 1 insertion(+), 3 deletions(-) - -James Hilliard in commit f0b1aee9e: - Use pkg-config to properly detect libssl and libcrypto libraries (#7452) - 1 file changed, 15 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit e92b99564: - TLS: Add missing redis-cli options. (#7456) - 3 files changed, 166 insertions(+), 52 deletions(-) - -Oran Agra in commit 1f3db5bf5: - redis-cli --hotkeys fixed to handle non-printable key names - 1 file changed, 11 insertions(+), 5 deletions(-) - -Oran Agra in commit c3044f369: - redis-cli --bigkeys fixed to handle non-printable key names - 1 file changed, 24 insertions(+), 16 deletions(-) - -Oran Agra in commit b3f75527b: - RESTORE ABSTTL won't store expired keys into the db (#7472) - 4 files changed, 46 insertions(+), 16 deletions(-) - -huangzhw in commit 6f87fc92f: - defrag.c activeDefragSdsListAndDict when defrag sdsele, We can't use (#7492) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit d8e6a3e5b: - skip a test that uses +inf on valgrind (#7440) - 1 file changed, 12 insertions(+), 9 deletions(-) - -Oran Agra in commit 28fd1a110: - stabilize tests that look for log lines (#7367) - 3 files changed, 33 insertions(+), 11 deletions(-) - -Oran Agra in commit a513b4ed9: - tests/valgrind: don't use debug restart (#7404) - 4 files changed, 114 insertions(+), 57 deletions(-) - -Oran Agra in commit 70e72fc1b: - change references to the github repo location (#7479) - 5 files changed, 7 insertions(+), 7 deletions(-) - -zhaozhao.zz in commit c63e533cc: - BITOP: propagate only when it really SET or DEL targetkey (#5783) - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 31040ff54: - Update comment to clarify change in #7398. - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit b605fe827: - LPOS: option FIRST renamed RANK. - 2 files changed, 19 insertions(+), 19 deletions(-) - -Dave Nielsen in commit 8deb24954: - updated copyright year - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit a61c2930c: - EXEC always fails with EXECABORT and multi-state is cleared - 6 files changed, 204 insertions(+), 91 deletions(-) - -antirez in commit 3c8041637: - Include cluster.h for getClusterConnectionsCount(). - 1 file changed, 1 insertion(+) - -antirez in commit 5be673ee8: - Fix BITFIELD i64 type handling, see #7417. - 1 file changed, 8 insertions(+), 6 deletions(-) - -antirez in commit 5f289df9b: - Clarify maxclients and cluster in conf. Remove myself too. - 2 files changed, 9 insertions(+), 1 deletion(-) - -hwware in commit 000f928d6: - fix memory leak in sentinel connection sharing - 1 file changed, 1 insertion(+) - -chenhui0212 in commit d9a3c0171: - Fix comments in function raxLowWalk of listpack.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -Tomasz Poradowski in commit 7526e4506: - ensure SHUTDOWN_NOSAVE in Sentinel mode - 2 files changed, 9 insertions(+), 8 deletions(-) - -chenhui0212 in commit 6487cbc33: - fix comments in listpack.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 69b66bfca: - Use cluster connections too, to limit maxclients. - 3 files changed, 23 insertions(+), 8 deletions(-) - -antirez in commit 5a960a033: - Tracking: fix enableBcastTrackingForPrefix() invalid sdslen() call. - 1 file changed, 1 insertion(+), 1 deletion(-) - -root in commit 1c2e50de3: - cluster.c remove if of clusterSendFail in markNodeAsFailingIfNeeded - 1 file changed, 1 insertion(+), 1 deletion(-) - -meir@redislabs.com in commit 040efb697: - Fix RM_ScanKey module api not to return int encoded strings - 3 files changed, 24 insertions(+), 7 deletions(-) - -antirez in commit 1b8b7941d: - Fix LCS object type checking. Related to #7379. - 1 file changed, 17 insertions(+), 10 deletions(-) - -hwware in commit 6b571b45a: - fix memory leak - 1 file changed, 11 insertions(+), 12 deletions(-) - -hwware in commit 674759062: - fix server crash in STRALGO command - 1 file changed, 7 insertions(+) - -Benjamin Sergeant in commit a05ffefdc: - Update redis-cli.c - 1 file changed, 19 insertions(+), 6 deletions(-) - -Jamie Scott in commit 870b63733: - minor fix - 1 file changed, 2 insertions(+), 3 deletions(-) - -================================================================================ -Redis 6.0.5 Released Tue Jun 09 11:56:08 CEST 2020 -================================================================================ - -Upgrade urgency MODERATE: several bugs with moderate impact are fixed here. - -The most important issues are listed here: - -* Fix handling of speical chars in ACL LOAD. -* Make Redis Cluster more robust about operation errors that may lead - to two clusters to mix together. -* Revert the sendfile() implementation of RDB transfer. It causes some delay. -* Fix TLS certificate loading for chained certificates. -* Fix AOF rewirting of KEEPTTL SET option. -* Fix MULTI/EXEC behavior during -BUSY script errors. - -And this is the full list of commits: - -antirez in commit ee8dd01bb: - Temporary fix for #7353 issue about EVAL during -BUSY. - 1 file changed, 9 insertions(+) - -xhe in commit a4a856d53: - return the correct proto version HELLO should return the current proto version, while the code hardcoded 3 - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit e2046b300: - Don't queue commands in an already aborted MULTI state - 1 file changed, 7 insertions(+) - -Oran Agra in commit b35fdf1de: - Avoid rejecting WATCH / UNWATCH, like MULTI/EXEC/DISCARD - 1 file changed, 4 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit 1d7bf208c: - AOF: append origin SET if no expire option - 2 files changed, 23 insertions(+), 8 deletions(-) - -Oran Agra in commit 676445ad9: - fix disconnectSlaves, to try to free each slave. - 1 file changed, 1 deletion(-) - -zhaozhao.zz in commit 4846c0c8a: - donot free protected client in freeClientsInAsyncFreeQueue - 1 file changed, 9 insertions(+), 3 deletions(-) - -Oran Agra in commit f33de403e: - fix pingoff test race - 1 file changed, 1 insertion(+) - -Kevin Fwu in commit 49af4d07e: - Fix TLS certificate loading for chained certificates. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 329fddbda: - Revert "Implements sendfile for redis." - 2 files changed, 2 insertions(+), 55 deletions(-) - -antirez in commit 925a2cd5a: - Revert "avoid using sendfile if tls-replication is enabled" - 1 file changed, 27 insertions(+), 34 deletions(-) - -Liu Zhen in commit 84a7a9058: - fix clusters mixing accidentally by gossip - 1 file changed, 10 insertions(+), 2 deletions(-) - -antirez in commit cd63359a1: - Fix handling of special chars in ACL LOAD. - 1 file changed, 8 insertions(+), 4 deletions(-) - -================================================================================ -Redis 6.0.4 Released Thu May 28 11:36:45 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: this release fixes a severe replication bug. - -Redis 6.0.4 fixes a critical replication bug caused by a new feature introduced -in Redis 6. The feature, called "meaningful offset" and strongly wanted by -myself (antirez) was an improvement that avoided that masters were no longer -able, during a failover where they were demoted to replicas, to partially -synchronize with the new master. In short the feature was able to avoid full -synchronizations with RDB. How did it work? By trimming the replication backlog -of the final "PING" commands the master was sending in the replication channel: -this way the replication offset would no longer go "after" the one of the -promoted replica, allowing the master to just continue in the same replication -history, receiving only a small data difference. - -However after the introduction of the feature we (the Redis core team) quickly -understood there was something wrong: the apparently harmless feature had -many bugs, and the last bug we discovered, after a joined effort of multiple -people, we were not even able to fully understand after fixing it. Enough was -enough, we decided that the complexity cost of this feature was too high. -So Redis 6.0.4 removes the feature entirely, and fixes the data corruption that -it was able to cause. - -However there are two facts to take in mind. - -Fact 1: Setups using chained replication, that means that certain replicas -are replicating from other replicas, up to Redis 6.0.3 can experience data -corruption. For chained replication we mean that: - - +--------+ +---------+ +-------------+ - | master |--------->| replica |-------->| sub-replica | - +--------+ +---------+ +-------------+ - - -People using chained replication SHOULD UPGRADE ASAP away from Redis 6.0.0, -6.0.1, 6.0.2 or 6.0.3 to Redis 6.0.4. - -To be clear, people NOT using this setup, but having just replicas attached -directly to the master, SHOUDL NOT BE in danger of any problem. But we -are no longer confident on 6.0.x replication implementation complexities -so we suggest to upgrade to 6.0.4 to everybody using an older 6.0.3 release. -We just so far didn't find any bug that affects Redis 6.0.3 that does not -involve chained replication. - -People starting with Redis 6.0.4 are fine. People with Redis 5 are fine. -People upgrading from Redis 5 to Redis 6.0.4 are fine. -TLDR: The problem is with users of 6.0.0, 6.0.1, 6.0.2, 6.0.3. - -Fact 2: Upgrading from Redis 6.0.x to Redis 6.0.4, IF AND ONLY IF you -use chained replication, requires some extra care: - -1. Once you attach your new Redis 6.0.4 instance as a replica of the current - Redis 6.0.x master, you should wait for the first full synchronization, - then you should promote it right away, if your setup involves chained - replication. Don't give it the time to do a new partial synchronization - in the case the link between the master and the replica will break in - the mean time. - -2. As an additional care, you may want to set the replication ping period - to a very large value (for instance 1000000) using the following command: - - CONFIG SET repl-ping-replica-period 1000000 - - Note that if you do "1" with care, "2" is not needed. - However if you do it, make sure to later restore it to its default: - - CONFIG SET repl-ping-replica-period 10 - -So this is the main change in Redis 6. Later we'll find a different way in -order to achieve what we wanted to achieve with the Meaningful Offset feature, -but without the same complexity. - -Other changes in this release: - -* PSYNC2 tests improved. -* Fix a rare active defrag edge case bug leading to stagnation -* Fix Redis 6 asserting at startup in 32 bit systems. -* Redis 6 32 bit is now added back to our testing environments. -* Fix server crash for STRALGO command, -* Implement sendfile for RDB transfer. -* TLS fixes. -* Make replication more resistant by disconnecting the master if we - detect a protocol error. Basically we no longer accept inline protocol - from the master. -* Other improvements in the tests. - -Regards, -antirez - -This is the full list of commits: - -antirez in commit 59cd4c9f6: - Test: take PSYNC2 test master timeout high during switch. - 1 file changed, 1 deletion(-) - -antirez in commit 6c1bb7b19: - Test: add the tracking unit as default. - 1 file changed, 1 insertion(+) - -Oran Agra in commit 1aee695e5: - tests: find_available_port start search from next port - 1 file changed, 12 insertions(+), 7 deletions(-) - -Oran Agra in commit a2ae46352: - tests: each test client work on a distinct port range - 5 files changed, 39 insertions(+), 27 deletions(-) - -Oran Agra in commit 86e562d69: - 32bit CI needs to build modules correctly - 2 files changed, 7 insertions(+), 2 deletions(-) - -Oran Agra in commit ab2984b1e: - adjust revived meaningful offset tests - 1 file changed, 39 insertions(+), 20 deletions(-) - -Oran Agra in commit 1ff5a222d: - revive meaningful offset tests - 2 files changed, 213 insertions(+) - -antirez in commit cc549b46a: - Replication: showLatestBacklog() refactored out. - 3 files changed, 36 insertions(+), 25 deletions(-) - -antirez in commit 377dd0515: - Drop useless line from replicationCacheMaster(). - 1 file changed, 2 deletions(-) - -antirez in commit 3f8d113f1: - Another meaningful offset test removed. - 1 file changed, 100 deletions(-) - -antirez in commit d4541349d: - Remove the PSYNC2 meaningful offset test. - 2 files changed, 113 deletions(-) - -antirez in commit 2112a5702: - Remove the meaningful offset feature. - 4 files changed, 10 insertions(+), 93 deletions(-) - -antirez in commit d2eb6e0b4: - Set a protocol error if master use the inline protocol. - 1 file changed, 17 insertions(+), 2 deletions(-) - -Oran Agra in commit 9c1df3b76: - daily CI test with tls - 1 file changed, 15 insertions(+) - -Oran Agra in commit 115ed1911: - avoid using sendfile if tls-replication is enabled - 1 file changed, 34 insertions(+), 27 deletions(-) - -antirez in commit 11c748aac: - Replication: log backlog creation event. - 1 file changed, 3 insertions(+) - -antirez in commit 8f1013722: - Test: PSYNC2 test can now show server logs. - 1 file changed, 88 insertions(+), 25 deletions(-) - -antirez in commit 2e591fc4a: - Clarify what is happening in PR #7320. - 1 file changed, 5 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit cbb51fb8f: - PSYNC2: second_replid_offset should be real meaningful offset - 1 file changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit e0fc88b4d: - add CI for 32bit build - 2 files changed, 34 insertions(+) - -antirez in commit e3f864b5f: - Make disconnectSlaves() synchronous in the base case. - 3 files changed, 20 insertions(+), 9 deletions(-) - -ShooterIT in commit 8af1e513f: - Implements sendfile for redis. - 2 files changed, 55 insertions(+), 2 deletions(-) - -antirez in commit 3c21418cd: - Fix #7306 less aggressively. - 2 files changed, 29 insertions(+), 17 deletions(-) - -Madelyn Olson in commit e201f83ce: - EAGAIN for tls during diskless load - 1 file changed, 4 insertions(+) - -Qu Chen in commit 58fc456cb: - Disconnect chained replicas when the replica performs PSYNC with the master always to avoid replication offset mismatch between master and chained replicas. - 2 files changed, 60 insertions(+), 3 deletions(-) - -hwware in commit 3febc5c29: - using moreargs variable - 1 file changed, 2 insertions(+), 2 deletions(-) - -hwware in commit 8d6738559: - fix server crash for STRALGO command - 1 file changed, 2 insertions(+), 2 deletions(-) - -ShooterIT in commit 7a35eec54: - Replace addDeferredMultiBulkLength with addReplyDeferredLen in comment - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit f93e1417b: - TLS: Improve tls-protocols clarity in redis.conf. - 1 file changed, 3 insertions(+), 2 deletions(-) - -ShooterIT in commit d0c9e4454: - Fix reply bytes calculation error - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 1cde6a060: - Tracking: flag CLIENT_TRACKING_BROKEN_REDIR when redir broken - 1 file changed, 1 insertion(+) - -Oran Agra in commit 436be3498: - fix a rare active defrag edge case bug leading to stagnation - 4 files changed, 146 insertions(+), 23 deletions(-) - -Oran Agra in commit f9d2ffdc5: - improve DEBUG MALLCTL to be able to write to write only fields. - 1 file changed, 27 insertions(+), 7 deletions(-) - -hujie in commit d7968ee92: - fix clear USER_FLAG_ALLCOMMANDS flag in acl - 1 file changed, 5 insertions(+), 4 deletions(-) - -ShooterIT in commit a902e6b25: - Redis Benchmark: generate random test data - 1 file changed, 12 insertions(+), 1 deletion(-) - -hwware in commit 9564ed7c3: - Redis-Benchmark: avoid potentical memmory leaking - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 2e4182743: - Handle keys with hash tag when computing hash slot using tcl cluster client. - 1 file changed, 23 insertions(+), 2 deletions(-) - -WuYunlong in commit eb2c8b2c6: - Add a test to prove current tcl cluster client can not handle keys with hash tag. - 1 file changed, 7 insertions(+), 1 deletion(-) - -ShooterIT in commit 928e6976b: - Use dictSize to get the size of dict in dict.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -Madelyn Olson in commit cdcf5af5a: - Converge hash validation for adding and removing - 1 file changed, 21 insertions(+), 14 deletions(-) - -Benjamin Sergeant in commit e8b09d220: - do not handle --cluster-yes for cluster fix mode - 1 file changed, 16 insertions(+), 7 deletions(-) - -Benjamin Sergeant in commit 57b4fb0d8: - fix typo ... - 1 file changed, 1 insertion(+), 1 deletion(-) - -Benjamin Sergeant in commit 29f25e411: - Redis-cli 6.0.1 `--cluster-yes` doesn't work (fix #7246) - 1 file changed, 5 insertions(+), 1 deletion(-) - -Oran Agra in commit 00d8b92b8: - fix valgrind test failure in replication test - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 5e17e6276: - add regression test for the race in #7205 - 1 file changed, 52 insertions(+) - -antirez in commit 96e7c011e: - Improve the PSYNC2 test reliability. - 1 file changed, 33 insertions(+), 15 deletions(-) - -================================================================================ -Redis 6.0.3 Released Sat May 16 18:10:21 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: a crash introduced in 6.0.2 is now fixed. - -1eab62f7e Remove the client from CLOSE_ASAP list before caching the master. - -================================================================================ -Redis 6.0.2 Released Fri May 15 22:24:36 CEST 2020 -================================================================================ - -Upgrade urgency MODERATE: many not critical bugfixes in different areas. - Critical fix to client side caching when - keys are evicted from the tracking table but - no notifications are sent. - -The following are the most serious fix: - -* XPENDING should not update consumer's seen-time -* optimize memory usage of deferred replies - fixed -* Fix CRC64 initialization outside the Redis server itself. -* stringmatchlen() should not expect null terminated strings. -* Cluster nodes availability checks improved when there is - high Pub/Sub load on the cluster bus. -* Redis Benchmark: Fix coredump because of double free -* Tracking: send eviction messages when evicting entries. -* rax.c updated from upstream antirez/rax. -* fix redis 6.0 not freeing closed connections during loading. - -New features: - -* Support setcpuaffinity on linux/bsd -* Client Side Caching: Add Tracking Prefix Number Stats in Server Info -* Add --user argument to redis-benchmark.c (ACL) - -Full list of commits: - -Yossi Gottlieb in commit 16ba33c05: - TLS: Fix test failures on recent Debian/Ubuntu. - 1 file changed, 20 deletions(-) - -Yossi Gottlieb in commit 77ae66930: - TLS: Add crypto locks for older OpenSSL support. - 1 file changed, 45 insertions(+) - -David Carlier in commit 389697988: - NetBSD build update. - 3 files changed, 30 insertions(+), 1 deletion(-) - -Madelyn Olson in commit 2435341d7: - Added a refcount on timer events to prevent deletion of recursive timer calls - 2 files changed, 12 insertions(+) - -antirez in commit 80c906bd3: - Cache master without checking of deferred close flags. - 3 files changed, 11 insertions(+), 8 deletions(-) - -antirez in commit 74249be4a: - Track events processed while blocked globally. - 5 files changed, 32 insertions(+), 17 deletions(-) - -antirez in commit 8bf660af9: - Some rework of #7234. - 4 files changed, 77 insertions(+), 65 deletions(-) - -Oran Agra in commit 9da134cd8: - fix redis 6.0 not freeing closed connections during loading. - 3 files changed, 133 insertions(+), 58 deletions(-) - -antirez in commit f7f219a13: - Regression test for #7249. - 1 file changed, 22 insertions(+) - -antirez in commit 693629585: - rax.c updated from upstream antirez/rax. - 1 file changed, 4 insertions(+), 2 deletions(-) - -antirez in commit e3b5648df: - Tracking: send eviction messages when evicting entries. - 2 files changed, 29 insertions(+), 12 deletions(-) - -Oran Agra in commit 5c41802d5: - fix unstable replication test - 1 file changed, 2 insertions(+), 2 deletions(-) - -ShooterIT in commit a23cdbb94: - Redis Benchmark: Fix coredump because of double free - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 1276058ea: - Cluster: clarify we always resolve the sender. - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit 002fcde3d: - Cluster: refactor ping/data delay handling. - 1 file changed, 13 insertions(+), 11 deletions(-) - -antirez in commit 960186a71: - Cluster: introduce data_received field. - 2 files changed, 27 insertions(+), 10 deletions(-) - -antirez in commit 3672875b4: - stringmatchlen() should not expect null terminated strings. - 1 file changed, 2 insertions(+), 2 deletions(-) - -Brad Dunbar in commit 24e12641d: - Remove unreachable branch. - 1 file changed, 2 deletions(-) - -hwware in commit c7edffbd5: - add jemalloc-bg-thread config in redis conf - 1 file changed, 3 insertions(+) - -hwware in commit 8a9c84f4a: - add include guard for lolwut.h - 1 file changed, 6 insertions(+) - -antirez in commit cb683a84f: - Don't propagate spurious MULTI on DEBUG LOADAOF. - 2 files changed, 6 insertions(+), 3 deletions(-) - -antirez in commit 84d9766d6: - Dump recent backlog on master query generating errors. - 1 file changed, 29 insertions(+) - -Titouan Christophe in commit ec1e106ec: - make struct user anonymous (only typedefed) - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit e48c37316: - Test: --dont-clean should do first cleanup. - 1 file changed, 2 insertions(+), 5 deletions(-) - -Benjamin Sergeant in commit 1e561cfaa: - Add --user argument to redis-benchmark.c (ACL) - 1 file changed, 15 insertions(+), 2 deletions(-) - -antirez in commit d1af82a88: - Drop not needed part from #7194. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Muhammad Zahalqa in commit 897a360d0: - Fix compiler warnings on function rev(unsigned long) - 1 file changed, 3 insertions(+), 3 deletions(-) - -antirez in commit ac316d8cc: - Move CRC64 initialization in main(). - 2 files changed, 1 insertion(+), 4 deletions(-) - -antirez in commit fc7bc3204: - Fix CRC64 initialization outside the Redis server itself. - 1 file changed, 3 insertions(+) - -hwware in commit a6e55c096: - Client Side Caching: Add Tracking Prefix Number Stats in Server Info - 3 files changed, 8 insertions(+) - -antirez in commit b062fd523: - Fix NetBSD build by fixing redis_set_thread_title() support. - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit 4efb25d9c: - Rework a bit the documentation for CPU pinning. - 2 files changed, 18 insertions(+), 8 deletions(-) - -zhenwei pi in commit d6436eb7c: - Support setcpuaffinity on linux/bsd - 12 files changed, 180 insertions(+), 1 deletion(-) - -Guy Benoish in commit 3a441c7d9: - XPENDING should not update consumer's seen-time - 4 files changed, 33 insertions(+), 20 deletions(-) - -Oran Agra in commit 75addb4fe: - optimize memory usage of deferred replies - fixed - 1 file changed, 29 insertions(+) - -Deliang Yang in commit c57d9146f: - reformat code - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 3d3861dd8: - add daily github actions with libc malloc and valgrind - 5 files changed, 106 insertions(+), 18 deletions(-) - - -================================================================================ -Redis 6.0.1 Released Sat May 02 00:06:07 CEST 2020 -================================================================================ - -Upgrade urgency HIGH: This release fixes a crash when builiding against - Libc malloc. - -Here we revert 8110ba888, an optimization that causes a crash due to a -bug in the code. It does not happen with the default allocator because of -differences between Jemalloc and libc malloc, so this escaped all our -testing but was reported by a user. We'll add back the original optimization -that was reverted here later, after checking what happens: it is not a -critical optimization. - -The other commits are minor stuff: - -antirez in commit db73d0998: - Cast printf() argument to the format specifier. - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit 7c0fe7271: - Revert "optimize memory usage of deferred replies" - 1 file changed, 31 deletions(-) - -antirez in commit 8fe25edc7: - Save a call to stopThreadedIOIfNeeded() for the base case. - 1 file changed, 3 insertions(+), 3 deletions(-) - -================================================================================ -Redis 6.0.0 GA Released Thu Apr 30 14:55:02 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: many bugs fixed compared to the last release - candidate. Better to upgrade if you see things - affecting your environment in the changelog. - -Hi all, finally we have Redis 6.0.0 GA! Enjoy this new Redis release. -Most of the documentation was updated today so that you can likely -find what you are looking for about the new features at redis.io. -This is the list of what changed compared to the previoius release candidate: - -* XCLAIM AOF/replicas propagation fixed. -* Client side caching: new NOLOOP option to avoid getting notified about - changes performed by ourselves. -* ACL GENPASS now uses HMAC-SHA256 and have an optional "bits" argument. - It means you can use it as a general purpose "secure random strings" - primitive! -* Cluster "SLOTS" subcommand memory optimization. -* The LCS command is now a subcommand of STRALGO. -* Meaningful offset for replicas as well. More successful partial - resynchronizations. -* Optimize memory usage of deferred replies. -* Faster CRC64 algorithm for faster RDB loading. -* XINFO STREAM FULL, a new subcommand to get the whole stream state. -* CLIENT KILL USER . -* MIGRATE AUTH2 option, for ACL style authentication support. -* Other random bugfixes. - -Enjoy Redis 6! :-) -Goodbye antirez - -List of commits in this release: - -antirez in commit 1f9b82bd5: - Update help.h again before Redis 6 GA. - 1 file changed, 17 insertions(+), 12 deletions(-) - -antirez in commit 3fcffe7d0: - redis-cli: fix hints with subcommands. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 455d8a05c: - redis-cli command help updated. - 1 file changed, 165 insertions(+), 25 deletions(-) - -zhaozhao.zz in commit 70287bbc9: - lazyfree & eviction: record latency generated by lazyfree eviction - 1 file changed, 18 insertions(+), 13 deletions(-) - -antirez in commit 7be21139a: - MIGRATE AUTH2 for ACL support. - 1 file changed, 19 insertions(+), 5 deletions(-) - -antirez in commit e1ee1a49d: - CLIENT KILL USER . - 1 file changed, 11 insertions(+) - -antirez in commit d56f058c0: - Fix tracking table max keys option in redis.conf. - 1 file changed, 12 insertions(+), 9 deletions(-) - -antirez in commit 96dd5fc93: - redis-cli: safer cluster fix with unreachalbe masters. - 1 file changed, 26 insertions(+), 1 deletion(-) - -antirez in commit 5b59d9c5d: - redis-cli: simplify cluster nodes coverage display. - 1 file changed, 10 insertions(+), 17 deletions(-) - -antirez in commit c163d4add: - redis-cli: try to make clusterManagerFixOpenSlot() more readable. - 1 file changed, 25 insertions(+), 6 deletions(-) - -Guy Benoish in commit aab74b715: - XINFO STREAM FULL should have a default COUNT of 10 - 1 file changed, 8 insertions(+), 4 deletions(-) - -antirez in commit 606134f9d: - Comment clearly why we moved some code in #6623. - 1 file changed, 4 insertions(+), 1 deletion(-) - -srzhao in commit ee627bb66: - fix pipelined WAIT performance issue. - 1 file changed, 13 insertions(+), 13 deletions(-) - -antirez in commit 47b8a7f9b: - Fix create-cluster BIN_PATH. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Benoish in commit 6c0bc608a: - Extend XINFO STREAM output - 2 files changed, 226 insertions(+), 34 deletions(-) - -hwware in commit 5bfc18950: - Fix not used marco in cluster.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit 56d628f85: - Update create-cluster - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit cac9d7cf7: - Adds `BIN_PATH` to create-cluster - 1 file changed, 8 insertions(+), 6 deletions(-) - -Oran Agra in commit b712fba17: - hickup, re-fix dictEncObjKeyCompare - 1 file changed, 4 insertions(+), 4 deletions(-) - -Oran Agra in commit ea63aea72: - fix loading race in psync2 tests - 3 files changed, 15 insertions(+), 1 deletion(-) - -antirez in commit 64e588bfa: - Rework comment in dictEncObjKeyCompare(). - 1 file changed, 8 insertions(+), 9 deletions(-) - -Oran Agra in commit 0d1e8c93b: - allow dictFind using static robj - 1 file changed, 9 insertions(+), 4 deletions(-) - -Madelyn Olson in commit a1bed447b: - Added crcspeed library - 2 files changed, 341 insertions(+) - -Madelyn Olson in commit a75fa3aad: - Made crc64 test consistent - 1 file changed, 3 insertions(+), 2 deletions(-) - -Madelyn Olson in commit 52c75e9db: - Implemented CRC64 based on slice by 4 - 5 files changed, 124 insertions(+), 157 deletions(-) - -Oran Agra in commit 8110ba888: - optimize memory usage of deferred replies - 1 file changed, 31 insertions(+) - -Oran Agra in commit e4d2bb62b: - Keep track of meaningful replication offset in replicas too - 5 files changed, 212 insertions(+), 92 deletions(-) - -antirez in commit fea9788cc: - Fix STRALGO command flags. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Dave-in-lafayette in commit 2144047e1: - fix for unintended crash during panic response - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Benoish in commit 43329c9b6: - Add the stream tag to XSETID tests - 1 file changed, 1 insertion(+), 1 deletion(-) - -Dave-in-lafayette in commit 1e17d3de7: - fix for crash during panic before all threads are up - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3722f89f4: - LCS -> STRALGO LCS. - 4 files changed, 28 insertions(+), 15 deletions(-) - -antirez in commit 373ae6061: - Also use propagate() in streamPropagateGroupID(). - 1 file changed, 11 insertions(+), 1 deletion(-) - -yanhui13 in commit f03f1fad6: - add tcl test for cluster slots - 1 file changed, 44 insertions(+) - -yanhui13 in commit 374ffdf1c: - optimize the output of cluster slots - 1 file changed, 7 insertions(+), 4 deletions(-) - -antirez in commit 4db38d2ef: - Minor aesthetic changes to #7135. - 1 file changed, 5 insertions(+), 7 deletions(-) - -Valentino Geron in commit f0a261448: - XREADGROUP with NOACK should propagate only one XGROUP SETID command - 1 file changed, 13 insertions(+), 7 deletions(-) - -antirez in commit fbdef6a9b: - ACL: re-enable command execution of disabled users. - 1 file changed, 4 deletions(-) - -antirez in commit 05a41da75: - getRandomBytes(): use HMAC-SHA256. - 1 file changed, 30 insertions(+), 10 deletions(-) - -antirez in commit 345c3768d: - ACL GENPASS: take number of bits as argument. - 1 file changed, 21 insertions(+), 6 deletions(-) - -antirez in commit 639c8a1d9: - ACL GENPASS: emit 256 bits instead of 128. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 321acea03: - ACL: deny commands execution of disabled users. - 1 file changed, 4 insertions(+) - -Theo Buehler in commit b0920e6e8: - TLS: Fix build with SSL_OP_NO_CLIENT_RENEGOTIATION - 1 file changed, 1 insertion(+), 1 deletion(-) - -Yossi Gottlieb in commit 149b658b5: - TLS: Fix build on older verisons of OpenSSL. - 1 file changed, 2 insertions(+) - -antirez in commit 06917e581: - Tracking: test expired keys notifications. - 1 file changed, 13 insertions(+) - -antirez in commit e434b2ce4: - Tracking: NOLOOP tests. - 1 file changed, 32 insertions(+) - -antirez in commit f3a172887: - Tracking: signal key as modified when evicting. - 1 file changed, 1 insertion(+) - -antirez in commit e63bb7ec8: - Tracking: NOLOOP further implementation and fixes. - 2 files changed, 21 insertions(+), 6 deletions(-) - -antirez in commit 6791ff052: - Tracking: NOLOOP internals implementation. - 17 files changed, 174 insertions(+), 112 deletions(-) - -antirez in commit 725b8cc68: - Implement redis_set_thread_title for MacOS. - 1 file changed, 6 insertions(+) - -zhenwei pi in commit 3575b8706: - Threaded IO: set thread name for redis-server - 3 files changed, 28 insertions(+) - -antirez in commit a76c67578: - Sentinel: small refactoring of sentinelCollectTerminatedScripts(). - 1 file changed, 1 insertion(+), 2 deletions(-) - -omg-by in commit 3a27064c4: - fix(sentinel): sentinel.running_scripts will always increase more times and not reset - 1 file changed, 1 insertion(+) - -antirez in commit 5c4c73e2c: - A few comments and name changes for #7103. - 1 file changed, 13 insertions(+), 4 deletions(-) - -Oran Agra in commit 6148f9493: - testsuite run the defrag latency test solo - 3 files changed, 42 insertions(+), 2 deletions(-) - -Jamie Scott in commit 51d3012d4: - Adding acllog-max-len to Redis.conf - 1 file changed, 9 insertions(+) - -antirez in commit c39f16c42: - Fix XCLAIM propagation in AOF/replicas for blocking XREADGROUP. - 2 files changed, 8 insertions(+), 3 deletions(-) - -================================================================================ -Redis 6.0-rc4 Released Thu Apr 16 16:10:35 CEST 2020 -================================================================================ - -Upgrade urgency LOW: If you are using RC3 without issues, don't rush. - -Hi all, this the latest release candidate of Redis 6. This is likely to -be very similar to what you'll see in Redis 6 GA. Please test it and -report any issue :-) - -Main changes in this release: - - * Big INFO speedup when using a lot of of clients. - * Big speedup on all the blocking commands: now blocking - on the same key is O(1) instead of being O(N). - * Stale replicas now allow MULTI/EXEC. - * New command: LCS (Longest Common Subsequence). - * Add a new configuration to make DEL like UNLINK. - * RDB loading speedup. - * Many bugs fixed (see the commit messages at the end of this node) - -See you in 14 days for Redis 6 GA. - -List of commits: - -antirez in commit 9f594e243: - Update SDS to latest version. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 48781dd95: - RESP3: fix HELLO map len in Sentinel mode. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 371ab0cff: - Don't allow empty spaces in ACL usernames. - 1 file changed, 36 insertions(+), 8 deletions(-) - -antirez in commit b86140ac5: - Don't allow empty spaces in ACL key patterns. - 1 file changed, 12 insertions(+), 1 deletion(-) - -liumiuyong in commit a7ee3c3e7: - FIX: truncate max/min longitude,latitude related geo_point (ex: {180, 85.05112878} ) - 1 file changed, 4 insertions(+) - -Guy Benoish in commit e5b9eb817: - Typo in getTimeoutFromObjectOrReply's error reply - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 0f31bb5c1: - Fix HELLO reply in Sentinel mode, see #6160. - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit b92d9a895: - fix spelling in acl.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 8f896e57a: - Fix zsetAdd() top comment spelling. - 1 file changed, 3 insertions(+), 3 deletions(-) - -hayleeliu in commit 8f5157058: - fix spelling mistake in bitops.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit ddeda9ceb: - Fix function names in zslDeleteNode() top comment. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit bde1f0a8e: - RESP3: change streams items from maps to arrays. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit bec68bff2: - Use the special static refcount for stack objects. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 0f239e51b: - RDB: refactor some RDB loading code into dbAddRDBLoad(). - 3 files changed, 22 insertions(+), 4 deletions(-) - -antirez in commit f855db61b: - incrRefCount(): abort on statically allocated object. - 2 files changed, 12 insertions(+), 2 deletions(-) - -antirez in commit 23094ba01: - More powerful DEBUG RELOAD. - 3 files changed, 55 insertions(+), 16 deletions(-) - -antirez in commit 8161a7a3e: - RDB: clarify a condition in rdbLoadRio(). - 2 files changed, 9 insertions(+), 2 deletions(-) - -antirez in commit 61b153073: - RDB: load files faster avoiding useless free+realloc. - 7 files changed, 40 insertions(+), 28 deletions(-) - -antirez in commit 414debfd0: - Speedup: unblock clients on keys in O(1). - 4 files changed, 50 insertions(+), 23 deletions(-) - -antirez in commit cbcd07777: - Fix ACL HELP table missing comma. - 1 file changed, 12 insertions(+), 12 deletions(-) - -mymilkbottles in commit 2437455f2: - Judge the log level in advance - 1 file changed, 1 insertion(+) - -antirez in commit 35c64b898: - Speedup INFO by counting client memory incrementally. - 4 files changed, 52 insertions(+), 26 deletions(-) - -qetu3790 in commit c3ac71748: - fix comments about RESIZE DB opcode in rdb.c - 1 file changed, 1 insertion(+), 4 deletions(-) - -antirez in commit c8dbcff9d: - Clarify redis.conf comment about lazyfree-lazy-user-del. - 1 file changed, 9 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit abd5156f2: - lazyfree: add a new configuration lazyfree-lazy-user-del - 4 files changed, 7 insertions(+), 2 deletions(-) - -antirez in commit 5719b3054: - LCS: more tests. - 1 file changed, 8 insertions(+) - -antirez in commit c89e1f293: - LCS: allow KEYS / STRINGS to be anywhere. - 1 file changed, 6 deletions(-) - -antirez in commit 0b16f8d44: - LCS tests. - 1 file changed, 22 insertions(+) - -antirez in commit 9254a805d: - LCS: get rid of STOREIDX option. Fix get keys helper. - 2 files changed, 20 insertions(+), 21 deletions(-) - -antirez in commit a4c490703: - LCS: fix stale comment. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit cb92c23de: - LCS: output LCS len as well in IDX mode. - 1 file changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 56a52e804: - LCS: MINMATCHLEN and WITHMATCHLEN options. - 1 file changed, 24 insertions(+), 11 deletions(-) - -antirez in commit ebb09a5c3: - LCS: 7x speedup by accessing the array with better locality. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit a9f8a8cba: - LCS: implement KEYS option. - 1 file changed, 18 insertions(+), 2 deletions(-) - -antirez in commit 4aa24e62a: - LCS: other fixes to range emission. - 1 file changed, 20 insertions(+), 16 deletions(-) - -antirez in commit 2b67b6b87: - LCS: fix emission of last range starting at index 0. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 420aac727: - LCS: implement range indexes option. - 1 file changed, 59 insertions(+), 9 deletions(-) - -antirez in commit a518a9a76: - LCS: initial functionality implemented. - 4 files changed, 156 insertions(+), 1 deletion(-) - -srzhao in commit 026cc11b0: - Check OOM at script start to get stable lua OOM state. - 3 files changed, 11 insertions(+), 4 deletions(-) - -Oran Agra in commit 02b594f6a: - diffrent fix for runtest --host --port - 2 files changed, 13 insertions(+), 13 deletions(-) - -Guy Benoish in commit f695d1830: - Try to fix time-sensitive tests in blockonkey.tcl - 1 file changed, 54 insertions(+), 1 deletion(-) - -Guy Benoish in commit 0e42cfc36: - Use __attribute__ only if __GNUC__ is defined - 1 file changed, 12 insertions(+), 3 deletions(-) - -Guy Benoish in commit 91ed9b3c4: - Modules: Perform printf-like format checks in variadic API - 1 file changed, 3 insertions(+), 3 deletions(-) - -Valentino Geron in commit 3e0d20962: - XREAD and XREADGROUP should not be allowed from scripts when BLOCK option is being used - 3 files changed, 18 insertions(+), 2 deletions(-) - -Guy Benoish in commit 240094c9b: - Stale replica should allow MULTI/EXEC - 1 file changed, 3 insertions(+), 3 deletions(-) - -Xudong Zhang in commit 209f3a1eb: - fix integer overflow - 1 file changed, 2 insertions(+), 2 deletions(-) - -Guy Benoish in commit 024c380b9: - Fix no-negative-zero test - 1 file changed, 1 insertion(+) - -Oran Agra in commit a38ff404b: - modules don't signalModifiedKey in setKey() since that's done (optionally) in RM_CloseKey - 4 files changed, 8 insertions(+), 8 deletions(-) - -Oran Agra in commit 814874d68: - change CI to build and run the module api tests - 1 file changed, 2 insertions(+) - -Oran Agra in commit 061616c1b: - fix possible warning on incomplete struct init - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Benoish in commit 7764996be: - Make sure Redis does not reply with negative zero - 2 files changed, 10 insertions(+) - -Guy Benoish in commit eba28e2ce: - DEBUG OBJECT should pass keyname to module when loading - 3 files changed, 4 insertions(+), 4 deletions(-) - -David Carlier in commit 15c9e79a7: - debug, dump registers on arm too. - 1 file changed, 55 insertions(+), 27 deletions(-) - -hwware in commit cd2b5df97: - fix spelling in cluster.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -Valentino Geron in commit 8cdc153f5: - XACK should be executed in a "all or nothing" fashion. - 2 files changed, 23 insertions(+), 1 deletion(-) - -hwware in commit b35407fa7: - add check for not switching between optin optout mode directly - 1 file changed, 12 insertions(+), 1 deletion(-) - -hwware in commit 4395889c9: - add check for not providing both optin optout flag - 1 file changed, 8 insertions(+) - -Guy Benoish in commit 1907e0f18: - PERSIST should notify a keyspace event - 1 file changed, 1 insertion(+) - -Guy Benoish in commit c35a53169: - streamReplyWithRange: Redundant XSETIDs to replica - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 6fe66e096: - Simplify comment in moduleTryServeClientBlockedOnKey(). - 1 file changed, 3 insertions(+), 12 deletions(-) - -Guy Benoish in commit 193fc241c: - Fix memory corruption in moduleHandleBlockedClients - 3 files changed, 149 insertions(+), 46 deletions(-) - -================================================================================ -Redis 6.0-rc3 Released Tue Mar 31 17:42:39 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: A connection management bug introduced with the - SSL implementation can crash Redis easily. - -Dear users, this is a list of the major changes in this release, please check -the list of commits for detail: - -* Fix crash due to refactoring for SSL, for the connection code. -* Precise timeouts for blocking commands. Now the timeouts have HZ - resolution regardless of the number of connected clinets. New timeouts - are stored in a radix tree and sorted by expire time. -* Fix rare crash when resizing the event loop because of CONFIG maxclients. -* Fix systemd readiness after successful partial resync. -* Redis-cli ask password mode to be prompted at startup (for additional safety). -* Keyspace notifications added to MIGRATE / RESTORE. -* Threaded I/O bugs fixed. -* Implement new ACL style AUTH in Sentinel. -* Make 'requirepass' more backward compatible with Redis <= 5. -* ACL: Handle default user as disabled if it's off regardless of "nopass". -* Fix a potential inconsistency when upgrading an instance in Redis Cluster - and restarting it. The instance will act as a replica but will actually be - set as a master immediately. However the choice of what to do with already - expired keys, on loading, was made from the POV of replicas. -* Abort transactions after -READONLY error. -* Many different fixes to module APIs. -* BITFIELD_RO added to call the command on read only replicas. -* PSYNC2: meaningful offset implementation. Allow the disconnected master - that is still sending PINGs to replicas, to be able to successfully - PSYNC incrementally to new slaves, discarding the last part of the - replication backlog consisting only of PINGs. -* Fix pipelined MULTI/EXEC during Lua scripts are in BUSY state. -* Re-fix propagation API in modules, broken again after other changes. - -antirez in commit ef1b1f01: - cast raxSize() to avoid warning with format spec. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 9f347fab: - Minor changes to #7037. - 2 files changed, 14 insertions(+), 5 deletions(-) - -Guy Benoish in commit a509400d: - Modules: Test MULTI/EXEC replication of RM_Replicate - 6 files changed, 49 insertions(+), 9 deletions(-) - -Guy Benoish in commit 805c8c94: - RENAME can unblock XREADGROUP - 3 files changed, 25 insertions(+), 1 deletion(-) - -antirez in commit 97b80b57: - Fix the propagate Tcl test after module changes. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 4f6b6b80: - Modify the propagate unit test to show more cases. - 1 file changed, 30 insertions(+), 2 deletions(-) - -antirez in commit 616b1cb7: - Fix module commands propagation double MULTI bug. - 4 files changed, 25 insertions(+), 8 deletions(-) - -antirez in commit 08fdef4b: - Fix RM_Call() stale comment due to cut&paste. - 1 file changed, 1 insertion(+), 3 deletions(-) - -OMG-By in commit 26b79ca1: - fix: dict.c->dictResize()->minimal type - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit fa418637: - PSYNC2: reset backlog_idx and master_repl_offset correctly - 1 file changed, 10 insertions(+), 5 deletions(-) - -antirez in commit bbbc80ac: - Precise timeouts: reference client pointer directly. - 1 file changed, 13 insertions(+), 16 deletions(-) - -antirez in commit c3b268a0: - timeout.c created: move client timeouts code there. - 5 files changed, 198 insertions(+), 167 deletions(-) - -Oran Agra in commit 0f7dfc37: - AOFRW on an empty stream created with MKSTREAM loads badkly - 2 files changed, 15 insertions(+), 1 deletion(-) - -antirez in commit 67643ead: - Precise timeouts: cleaup the table on unblock. - 3 files changed, 21 insertions(+), 2 deletions(-) - -antirez in commit ad94066e: - Precise timeouts: fix comments after functional change. - 2 files changed, 6 insertions(+), 6 deletions(-) - -antirez in commit a443ec2e: - Precise timeouts: use only radix tree for timeouts. - 3 files changed, 15 insertions(+), 38 deletions(-) - -antirez in commit 6862fd70: - Precise timeouts: fast exit for clientsHandleShortTimeout(). - 1 file changed, 1 insertion(+) - -antirez in commit 30f1df8c: - Precise timeouts: fix bugs in initial implementation. - 2 files changed, 5 insertions(+), 1 deletion(-) - -antirez in commit 7add0f24: - Precise timeouts: working initial implementation. - 3 files changed, 110 insertions(+), 28 deletions(-) - -antirez in commit 9d6d1779: - Precise timeouts: refactor unblocking on timeout. - 2 files changed, 33 insertions(+), 13 deletions(-) - -antirez in commit 316a8f15: - PSYNC2: fix backlog_idx when adjusting for meaningful offset - 1 file changed, 3 insertions(+) - -伯成 in commit 11db53f8: - Boost up performance for redis PUB-SUB patterns matching - 3 files changed, 43 insertions(+), 11 deletions(-) - -antirez in commit e257f121: - PSYNC2: meaningful offset test. - 2 files changed, 62 insertions(+) - -antirez in commit 5f72f696: - PSYNC2: meaningful offset implemented. - 3 files changed, 40 insertions(+), 1 deletion(-) - -antirez in commit 8caa2714: - Explain why we allow transactions in -BUSY state. - 1 file changed, 9 insertions(+), 2 deletions(-) - -Oran Agra in commit e43cd831: - MULTI/EXEC during LUA script timeout are messed up - 2 files changed, 73 insertions(+) - -antirez in commit 34b89832: - Improve comments of replicationCacheMasterUsingMyself(). - 1 file changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 70a98a43: - Fix BITFIELD_RO test. - 2 files changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 8783304a: - Abort transactions after -READONLY error. Fix #7014. - 1 file changed, 1 insertion(+) - -antirez in commit ec9cf002: - Minor changes to BITFIELD_RO PR #6951. - 1 file changed, 9 insertions(+), 6 deletions(-) - -bodong.ybd in commit b3e4abf0: - Added BITFIELD_RO variants for read-only operations. - 4 files changed, 54 insertions(+), 1 deletion(-) - -antirez in commit 50f8f950: - Modules: updated function doc after #7003. - 1 file changed, 6 insertions(+), 1 deletion(-) - -Guy Benoish in commit f2f3dc5e: - Allow RM_GetContextFlags to work with ctx==NULL - 1 file changed, 16 insertions(+), 14 deletions(-) - -hwware in commit eb808879: - fix potentical memory leak in redis-cli - 1 file changed, 2 insertions(+) - -Yossi Gottlieb in commit cdcab0e8: - Fix crashes related to failed/rejected accepts. - 1 file changed, 6 insertions(+), 5 deletions(-) - -Yossi Gottlieb in commit 50dcd9f9: - Cluster: fix misleading accept errors. - 1 file changed, 4 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit 87dbd8f5: - Conns: Fix connClose() / connAccept() behavior. - 3 files changed, 48 insertions(+), 32 deletions(-) - -hwware in commit 81e8686c: - remove redundant Semicolon - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit c7524a7e: - clean CLIENT_TRACKING_CACHING flag when disabled caching - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit 2dd1ca6a: - add missing commands in cluster help - 1 file changed, 2 insertions(+), 1 deletion(-) - -artix in commit 95324b81: - Support Redis Cluster Proxy PROXY INFO command - 1 file changed, 5 insertions(+), 1 deletion(-) - -박승현 in commit 04c53fa1: - Update redis.conf - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 0578157d: - Fix master replica inconsistency for upgrading scenario. - 3 files changed, 9 insertions(+), 2 deletions(-) - -WuYunlong in commit 299f1d02: - Add 14-consistency-check.tcl to prove there is a data consistency issue. - 1 file changed, 87 insertions(+) - -antirez in commit 61b98f32: - Regression test for #7011. - 1 file changed, 7 insertions(+) - -antirez in commit 34ea2f4e: - ACL: default user off should not allow automatic authentication. - 2 files changed, 3 insertions(+), 2 deletions(-) - -antirez in commit cbbf9b39: - Sentinel: document auth-user directive. - 1 file changed, 12 insertions(+) - -antirez in commit 9c2e42dd: - ACL: Make Redis 6 more backward compatible with requirepass. - 4 files changed, 17 insertions(+), 15 deletions(-) - -antirez in commit d387f67d: - Sentinel: implement auth-user directive for ACLs. - 1 file changed, 38 insertions(+), 7 deletions(-) - -zhaozhao.zz in commit 7c078416: - Threaded IO: bugfix client kill may crash redis - 1 file changed, 11 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit 9cc7038e: - Threaded IO: handle pending reads clients ASAP after event loop - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit da8c7c49: - Example sentinel conf: document requirepass. - 1 file changed, 8 insertions(+) - -antirez in commit bdb338cf: - Aesthetic changes in PR #6989. - 1 file changed, 9 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit b3e03054: - Threaded IO: bugfix #6988 process events while blocked - 1 file changed, 5 insertions(+) - -antirez in commit e628f944: - Restore newline at the end of redis-cli.c - 1 file changed, 2 insertions(+), 1 deletion(-) - -chendianqiang in commit 5d4c4df3: - use correct list for moduleUnregisterUsedAPI - 1 file changed, 1 insertion(+), 1 deletion(-) - -guodongxiaren in commit da14982d: - string literal should be const char* - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit dc8885a1: - Adds keyspace notifications to migrate and restore - 1 file changed, 3 insertions(+), 1 deletion(-) - -bodong.ybd in commit bfb18e55: - Remove duplicate obj files in Makefile - 1 file changed, 2 insertions(+), 2 deletions(-) - -bodong.ybd in commit 76d57161: - Fix bug of tcl test using external server - 2 files changed, 8 insertions(+), 2 deletions(-) - -fengpf in commit 0e5820d8: - fix comments in latency.c - 2 files changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 916dd79f: - Update linenoise. - 1 file changed, 2 insertions(+), 1 deletion(-) - -lifubang in commit c0c67c9b: - add askpass mode - 1 file changed, 19 insertions(+), 1 deletion(-) - -lifubang in commit e1c29434: - update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 - 4 files changed, 59 insertions(+), 4 deletions(-) - -Jamie Scott in commit e5a063bc: - Remove default guidance in Redis.conf - 1 file changed, 1 insertion(+), 2 deletions(-) - -Jamie Scott in commit d28cbaf7: - Update Redis.conf to improve TLS usability - 1 file changed, 2 insertions(+), 1 deletion(-) - -Johannes Truschnigg in commit 23d5e8b8: - Signal systemd readiness atfer Partial Resync - 1 file changed, 4 insertions(+) - -Oran Agra in commit 61738154: - fix for flaky psync2 test - 1 file changed, 21 insertions(+) - -antirez in commit 70e0e499: - ae.c: fix crash when resizing the event loop. - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit b3e4aa67: - Fix release notes spelling mistake. - 1 file changed, 1 insertion(+), 1 deletion(-) - - -================================================================================ -Redis 6.0 RC2 Released Thu Mar 05 15:40:53 CET 2020 -================================================================================ - -Upgrade urgency MODERATE: Normal bugfixing release of a non-GA branch. - -Hi Redis users, Redis 6 is approaching and will be released 30th of April. -New release candidates will be released at the end of March, then another -one mid April, to finally reach the GA at the end of April. - -Redis 6 RC2 brings many fixes and new things, especially in the area of -client side caching. This is the list of big changes in this release. As -usually you can find the full list of commits at the end: - -New features and improvements: - -* ACL LOG: log denied commands, keys accesses and authentications. -* Client side caching redesigned. Now we use keys not caching slots. -* Client side caching: Broadcasting mode implemented. -* Client side caching: OPTIN/OPTOUT modes implemented. -* Remove RDB files used for replication in persistence-less instances (option). - -Fixes (only selected ones, see commits for all the fixes): - -* Different fixes to streams in edge cases. -* Fix duplicated CLIENT SETNAME reply because of RESP3 changes. -* Fix crash due to new active expire division by zero. -* Avoid sentinel changes promoted_slave to be its own replica. -* Fix bug on KEYS command where pattern starts with * followed by \x00. -* Threaded I/O: now the main thread is used as well to do I/O. -* Many fixes to modules APIs, and more to come in the next RCs. -* ld2string should fail if string contains \0 in the middle. -* Make the Redis test more reliable. -* Fix SPOP returning nil (see #4709). WARNING: API change. - -qetu3790 in commit 4af0d7fd: - Fix not used constant in lru_test_mode. - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit 6ef01878: - add missing file marco - 1 file changed, 5 insertions(+) - -ShooterIT in commit fe81d5c8: - Avoid compiler warnings - 1 file changed, 1 insertion(+) - -antirez in commit c2f01d7f: - RDB deletion: document it in example redis.conf. - 1 file changed, 13 insertions(+) - -antirez in commit 127e09bc: - Make sync RDB deletion configurable. Default to no. - 3 files changed, 22 insertions(+), 4 deletions(-) - -antirez in commit a20303c6: - Check that the file exists in removeRDBUsedToSyncReplicas(). - 1 file changed, 8 insertions(+), 4 deletions(-) - -antirez in commit 7a23b945: - Log RDB deletion in persistence-less instances. - 1 file changed, 15 insertions(+), 2 deletions(-) - -antirez in commit baaf869f: - Introduce bg_unlink(). - 1 file changed, 31 insertions(+), 3 deletions(-) - -antirez in commit be4bc1a5: - Remove RDB files used for replication in persistence-less instances. - 3 files changed, 56 insertions(+), 1 deletion(-) - -antirez in commit 07dc1b42: - Use a smaller getkeys global buffer. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 10e71b3d: - Optimize temporary memory allocations for getKeysFromCommand mechanism - 1 file changed, 31 insertions(+), 10 deletions(-) - -antirez in commit edc0ed14: - Modules: reformat RM_Scan() top comment a bit. - 1 file changed, 21 insertions(+), 12 deletions(-) - -antirez in commit c5319612: - Modules: more details in RM_Scan API top comment. - 1 file changed, 22 insertions(+), 6 deletions(-) - -Oran Agra in commit fff6b26a: - RM_Scan disable dict rehashing - 2 files changed, 21 insertions(+), 6 deletions(-) - -Guy Benoish in commit 65048460: - Add RM_CreateStringFromDouble - 2 files changed, 14 insertions(+) - -Oran Agra in commit 3144a278: - add no_auth to COMMAND INFO - 1 file changed, 1 insertion(+) - -Oran Agra in commit afe0b16c: - module api docs for aux_save and aux_load - 2 files changed, 7 insertions(+), 1 deletion(-) - -Guy Benoish in commit df152b0c: - streamReplyWithRangeFromConsumerPEL: Redundant streamDecodeID - 1 file changed, 1 insertion(+), 3 deletions(-) - -antirez in commit e3c1f439: - Show Redis version when not understanding a config directive. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 141c0679: - Changelog: explain Redis 6 SPOP change. - 1 file changed, 4 insertions(+), 1 deletion(-) - -bodong.ybd in commit fe902461: - Fix spop return nil #4709 - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 9d4219eb: - Fix SDS misuse in enumConfigSet(). Related to #6778. - 1 file changed, 3 insertions(+), 3 deletions(-) - -antirez in commit 84243064: - Remove useless comment from enumConfigSet(). - 1 file changed, 1 deletion(-) - -Ponnuvel Palaniyappan in commit dafb94db: - Fix a potential overflow with strncpy - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit ea697b63: - Improve aeDeleteEventLoop() top comment grammar. - 1 file changed, 2 insertions(+), 1 deletion(-) - -wangyuan21 in commit dd479880: - free time event when delete eventloop - 1 file changed, 7 insertions(+) - -srzhao in commit ecf3b2ef: - fix impl of aof-child whitelist SIGUSR1 feature. - 1 file changed, 5 insertions(+), 4 deletions(-) - -meir@redislabs.com in commit 2966132c: - Changed log level for module fork api from 'notice' to 'verbos'. - 1 file changed, 2 insertions(+), 2 deletions(-) - -hwware in commit 7277e5d8: - format fix - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit 1bb5ee9c: - fix potentical memory leaks - 1 file changed, 4 insertions(+), 1 deletion(-) - -Hengjian Tang in commit 97329733: - modify the read buf size according to the write buf size PROTO_IOBUF_LEN defined before - 1 file changed, 1 insertion(+), 1 deletion(-) - -Ariel in commit 15ea1324: - fix ThreadSafeContext lock/unlock function names - 1 file changed, 2 insertions(+), 2 deletions(-) - -Guy Benoish in commit 4d12c37c: - XREADGROUP should propagate XCALIM/SETID in MULTI/EXEC - 1 file changed, 2 insertions(+), 2 deletions(-) - -Oran Agra in commit 12626ce9: - fix race in module api test for fork - 2 files changed, 2 insertions(+), 3 deletions(-) - -Guy Benoish in commit 2ecab0b6: - Modules: Do not auto-unblock clients if not blocked on keys - 1 file changed, 22 insertions(+), 7 deletions(-) - -Oran Agra in commit 635321d4: - fix github actions failing latency test for active defrag - part 2 - 2 files changed, 5 insertions(+), 4 deletions(-) - -Oran Agra in commit 0b988fa9: - fix github actions failing latency test for active defrag - 2 files changed, 14 insertions(+), 13 deletions(-) - -Oran Agra in commit 60096bc1: - Fix latency sensitivity of new defrag test - 1 file changed, 32 insertions(+), 8 deletions(-) - -antirez in commit b4395426: - Tracking: optin/out implemented. - 3 files changed, 82 insertions(+), 16 deletions(-) - -antirez in commit ef3551d1: - Test engine: experimental change to avoid busy port problems. - 1 file changed, 84 insertions(+), 49 deletions(-) - -antirez in commit 72c05351: - Test engine: detect timeout when checking for Redis startup. - 1 file changed, 11 insertions(+), 1 deletion(-) - -antirez in commit 294c9af4: - Test engine: better tracking of what workers are doing. - 2 files changed, 12 insertions(+), 4 deletions(-) - -hwware in commit ba027079: - add missing subcommand description for debug oom - 1 file changed, 1 insertion(+) - -Guy Benoish in commit 5d0890c0: - Fix memory leak in test_ld_conv - 1 file changed, 4 insertions(+) - -Madelyn Olson in commit d1f22eac: - Give an error message if you specify redirect twice - 1 file changed, 7 insertions(+) - -Madelyn Olson in commit 762fbcb6: - Minor CSC fixes and fixed documentation - 2 files changed, 16 insertions(+), 17 deletions(-) - -Oran Agra in commit 349aa245: - Defrag big lists in portions to avoid latency and freeze - 4 files changed, 350 insertions(+), 34 deletions(-) - -Guy Benoish in commit b4ddc7b7: - XGROUP DESTROY should unblock XREADGROUP with -NOGROUP - 2 files changed, 11 insertions(+) - -hayashier in commit 73806f74: - fix typo from fss to rss - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit b6129f86: - Test is more complex now, increase default timeout. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit f15fb727: - Tracking: fix max-keys configuration directive. - 2 files changed, 2 insertions(+), 2 deletions(-) - -Itamar Haber in commit e374573f: - Fixes segfault on calling trackingGetTotalKeys - 1 file changed, 1 insertion(+) - -antirez in commit 73d47d57: - Signal key as modified when expired on-access. - 1 file changed, 4 insertions(+), 2 deletions(-) - -antirez in commit b7cb28d5: - Tracking: first set of tests for the feature. - 1 file changed, 66 insertions(+) - -antirez in commit 1db72571: - Tracking: fix operators precedence error in bcast check. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit fe96e29d: - Tracking: fix behavior when switchinig from normal to BCAST. - 1 file changed, 11 insertions(+), 1 deletion(-) - -antirez in commit f21be1ec: - Tracking: fix sending messages bug + tracking off bug. - 2 files changed, 28 insertions(+), 20 deletions(-) - -antirez in commit 6fb1aa23: - Tracking: BCAST: basic feature now works. - 3 files changed, 55 insertions(+), 40 deletions(-) - -antirez in commit d4fe79a1: - Tracking: BCAST: broadcasting of keys in prefixes implemented. - 2 files changed, 94 insertions(+), 9 deletions(-) - -antirez in commit abb81c63: - Tracking: BCAST: registration in the prefix table. - 3 files changed, 67 insertions(+), 20 deletions(-) - -antirez in commit 77da9608: - Tracking: BCAST: parsing of the options + skeleton. - 4 files changed, 73 insertions(+), 19 deletions(-) - -antirez in commit 3e8c69a9: - Tracking: always reply with an array of keys. - 2 files changed, 10 insertions(+), 3 deletions(-) - -antirez in commit a788c373: - Tracking: minor change of names and new INFO field. - 4 files changed, 11 insertions(+), 4 deletions(-) - -antirez in commit df838927: - Rax.c: populate data field after random walk. - 1 file changed, 1 insertion(+) - -antirez in commit 0517da36: - Tracking: rename INFO field with total items. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3c16d6b3: - Tracking: first conversion from hashing to key names. - 3 files changed, 84 insertions(+), 114 deletions(-) - -Oran Agra in commit 3b4f1477: - add no-slowlog option to RM_CreateCommand - 1 file changed, 3 insertions(+) - -Khem Raj in commit 5e762d84: - Mark extern definition of SDS_NOINIT in sds.h - 1 file changed, 1 insertion(+), 1 deletion(-) - -lifubang in commit 54f5499a: - correct help info for --user and --pass - 1 file changed, 2 insertions(+), 2 deletions(-) - -Seunghoon Woo in commit 0c952b13: - [FIX] revisit CVE-2015-8080 vulnerability - 1 file changed, 6 insertions(+), 4 deletions(-) - -Guy Benoish in commit dd34f703: - Diskless-load emptyDb-related fixes - 3 files changed, 44 insertions(+), 28 deletions(-) - -lifubang in commit 5e042dbc: - fix ssl flag check for redis-cli - 1 file changed, 10 insertions(+), 9 deletions(-) - -Guy Benoish in commit dcbe8bfa: - Exclude "keymiss" notification from NOTIFY_ALL - 5 files changed, 12 insertions(+), 7 deletions(-) - -Oran Agra in commit 36caf2e4: - update RM_SignalModifiedKey doc comment - 1 file changed, 2 insertions(+), 1 deletion(-) - -Oran Agra in commit 3067352a: - Add handling of short read of module id in rdb - 1 file changed, 4 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 9baaf858: - TLS: Update documentation. - 2 files changed, 32 insertions(+), 31 deletions(-) - -Oran Agra in commit 4440133e: - A few non-data commands that should be allowed while loading or stale - 1 file changed, 8 insertions(+), 8 deletions(-) - -Oran Agra in commit c9577941: - Memory leak when bind config is provided twice - 1 file changed, 4 insertions(+) - -Oran Agra in commit 1333a46b: - fix maxmemory config warning - 1 file changed, 3 insertions(+), 2 deletions(-) - -Oran Agra in commit 8e7282eb: - Fix client flags to be int64 in module.c - 1 file changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit a678390e: - moduleRDBLoadError, add key name, and use panic rather than exit - 1 file changed, 5 insertions(+), 4 deletions(-) - -Oran Agra in commit 919fbf42: - reduce repeated calls to use_diskless_load - 1 file changed, 3 insertions(+), 4 deletions(-) - -Oran Agra in commit 22e45d46: - freeClientAsync don't lock mutex if there's just one thread - 1 file changed, 6 insertions(+), 1 deletion(-) - -Oran Agra in commit ba289244: - move restartAOFAfterSYNC from replicaofCommand to replicationUnsetMaster - 1 file changed, 4 insertions(+), 3 deletions(-) - -Oran Agra in commit f42ce57d: - stopAppendOnly resets aof_rewrite_scheduled - 1 file changed, 1 insertion(+) - -Oran Agra in commit df096bc9: - add SAVE subcommand to ACL HELP and top comment - 1 file changed, 2 insertions(+) - -Oran Agra in commit a55e5847: - DEBUG HELP - add PROTOCOL - 1 file changed, 3 insertions(+), 2 deletions(-) - -Guy Benoish in commit 5a6cfbf4: - Some refactroing using getClientType instead of CLIENT_SLAVE - 2 files changed, 18 insertions(+), 26 deletions(-) - -Guy Benoish in commit fae306b3: - Fix small bugs related to replica and monitor ambiguity - 2 files changed, 8 insertions(+), 6 deletions(-) - -Yossi Gottlieb in commit 73630966: - TLS: Some redis.conf clarifications. - 1 file changed, 10 insertions(+), 11 deletions(-) - -Oran Agra in commit 488e1947: - config.c verbose error replies for CONFIG SET, like config file parsing - 1 file changed, 31 insertions(+), 97 deletions(-) - -Oran Agra in commit c82ccf06: - memoryGetKeys helper function so that ACL can limit access to keys for MEMORY command - 3 files changed, 18 insertions(+), 1 deletion(-) - -antirez in commit 51c1a9f8: - ACL LOG: make max log entries configurable. - 4 files changed, 19 insertions(+) - -antirez in commit ea1e1b12: - ACL LOG: test for AUTH reason. - 1 file changed, 9 insertions(+) - -antirez in commit 7379c78a: - ACL LOG: log failed auth attempts. - 5 files changed, 34 insertions(+), 12 deletions(-) - -antirez in commit 9f6e84f6: - ACL LOG: implement a few basic tests. - 1 file changed, 87 insertions(+) - -antirez in commit 82790e51: - ACL LOG: also log ACL errors in the scripting/MULTI ctx. - 2 files changed, 6 insertions(+), 2 deletions(-) - -antirez in commit 943008eb: - ACL LOG: implement LOG RESET. - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit e271a611: - ACL LOG: group similar entries in a given time delta. - 1 file changed, 58 insertions(+), 3 deletions(-) - -antirez in commit f1974d5d: - ACL LOG: actually emit entries. - 3 files changed, 34 insertions(+), 5 deletions(-) - -antirez in commit d9b153c9: - ACL LOG: implement ACL LOG subcommadn skeleton. - 1 file changed, 37 insertions(+) - -antirez in commit 577fc438: - ACL LOG: data structures and initial functions. - 5 files changed, 54 insertions(+), 5 deletions(-) - -Leo Murillo in commit f7a94526: - Set ZSKIPLIST_MAXLEVEL to optimal value given 2^64 elements and p=0.25 - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit eecfa979: - Fix lua related memory leak. - 1 file changed, 1 insertion(+) - -WuYunlong in commit d2509811: - Add tcl regression test in scripting.tcl to reproduce memory leak. - 1 file changed, 5 insertions(+) - -Yossi Gottlieb in commit 29d4a150: - TLS: Fix missing initialization in redis-cli. - 1 file changed, 9 insertions(+) - -Oran Agra in commit ec0c61da: - fix uninitialized info_cb var in module.c - 1 file changed, 1 insertion(+) - -Guy Benoish in commit 6fe55c2f: - ld2string should fail if string contains \0 in the middle - 5 files changed, 20 insertions(+), 11 deletions(-) - -antirez in commit bbce3ba9: - Add more info in the unblockClientFromModule() function. - 1 file changed, 7 insertions(+), 1 deletion(-) - -Guy Benoish in commit 40295fb3: - Modules: Fix blocked-client-related memory leak - 3 files changed, 51 insertions(+), 6 deletions(-) - -antirez in commit 8e9d19bc: - Change error message for #6775. - 1 file changed, 2 insertions(+), 2 deletions(-) - -Vasyl Melnychuk in commit ba146d4c: - Make error when submitting command in incorrect context more explicit - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit 721a39dd: - Document I/O threads in redis.conf. - 1 file changed, 46 insertions(+) - -antirez in commit 5be3a15a: - Setting N I/O threads should mean N-1 additional + 1 main thread. - 1 file changed, 25 insertions(+), 22 deletions(-) - -antirez in commit cbabf779: - Simplify #6379 changes. - 2 files changed, 4 insertions(+), 9 deletions(-) - -WuYunlong in commit 658749cc: - Free allocated sds in pfdebugCommand() to avoid memory leak. - 1 file changed, 1 insertion(+) - -WuYunlong in commit 47988c96: - Fix potential memory leak of clusterLoadConfig(). - 1 file changed, 20 insertions(+), 5 deletions(-) - -WuYunlong in commit cc90f79b: - Fix potential memory leak of rioWriteBulkStreamID(). - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit ecd17e81: - Jump to right label on AOF parsing error. - 1 file changed, 6 insertions(+), 4 deletions(-) - -antirez in commit 1927932b: - Port PR #6110 to new connection object code. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit f2df5773: - A few comments about main thread serving I/O as well. - 1 file changed, 7 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit b3ff8a4b: - Threaded IO: use main thread to handle read work - 1 file changed, 8 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit b1f2c510: - Threaded IO: use main thread to handle write work - 1 file changed, 10 insertions(+), 2 deletions(-) - -ShooterIT in commit 7bbafc56: - Rename rdb asynchronously - 1 file changed, 7 insertions(+) - -Leo Murillo in commit c7f75266: - Fix bug on KEYS command where pattern starts with * followed by \x00 (null char). - 1 file changed, 1 insertion(+), 1 deletion(-) - -Jamie Scott in commit ed7ea13a: - Update to directive in redis.conf (missing s) - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3be77623: - Free fakeclient argv on AOF error. - 1 file changed, 11 insertions(+), 3 deletions(-) - -antirez in commit 15f6b748: - Git ignore: ignore more files. - 1 file changed, 2 insertions(+) - -Guy Benoish in commit 1b5bf40c: - Blocking XREAD[GROUP] should always reply with valid data (or timeout) - 3 files changed, 44 insertions(+), 10 deletions(-) - -John Sully in commit 954c20ed: - Add support for incremental build with header files - 2 files changed, 6 insertions(+), 1 deletion(-) - -WuYunlong in commit 11c3afd7: - Fix petential cluster link error. - 1 file changed, 4 insertions(+) - -Yossi Gottlieb in commit b752e83d: - Add REDISMODULE_CTX_FLAGS_MULTI_DIRTY. - 2 files changed, 8 insertions(+) - -hwware in commit e16eb874: - typo fix in acl.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -Itamar Haber in commit 35ea9d23: - Adjusts 'io_threads_num' max to 128 - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 38729126: - XCLAIM: Create the consumer only on successful claims. - 1 file changed, 4 insertions(+), 2 deletions(-) - -yz1509 in commit b9a15303: - avoid sentinel changes promoted_slave to be its own replica. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 5e7e5e6b: - Fix active expire division by zero. - 1 file changed, 7 insertions(+), 4 deletions(-) - -antirez in commit e61dde88: - Fix duplicated CLIENT SETNAME reply. - 1 file changed, 1 deletion(-) - -Guy Benoish in commit cddf1da2: - Stream: Handle streamID-related edge cases - 4 files changed, 54 insertions(+), 4 deletions(-) - -Oran Agra in commit 52ea44e5: - config.c adjust config limits and mutable - 2 files changed, 7 insertions(+), 7 deletions(-) - -antirez in commit 0f28ea16: - Inline protocol: handle empty strings well. - 1 file changed, 2 insertions(+), 6 deletions(-) - -antirez in commit 00e5fefe: - Fix ip and missing mode in RM_GetClusterNodeInfo(). - 1 file changed, 5 insertions(+), 2 deletions(-) - -================================================================================ -Redis 6.0 RC1 Released Thu Dec 19 09:58:24 CEST 2019 -================================================================================ - -Upgrade urgency LOW: This is the first RC of Redis 6. - -Introduction to the Redis 6 release -=================================== - -Redis 6 improves Redis in a number of key areas and is one of the largest -Redis releases in the history of the project, so here we'll list only -the biggest features in this release: - -* The modules system now has a number of new APIs that allow module authors - to make things otherwise not possible in the past. It is possible to - store arbitrary module private data in RDB files, to hook on different - server events, capture and rewrite commands executions, block clients on - keys, and so forth. -* The Redis active expire cycle was rewritten for much faster eviction of keys - that are already expired. Now the effort is tunable. -* Redis now supports SSL on all channels. -* ACL support, you can define users that can run only certain commands and/or - can only access only certain keys patterns. -* Redis now supports a new protocol called RESP3, which returns more - semantical replies: new clients using this protocol can understand just - from the reply what type to return to the calling program. -* There is server-side support for client-side caching of key values. This - feature is still experimental and will get more changes during the next - release candidates, but you can already test it and read about it here: - https://redis.io/topics/client-side-caching -* Redis can now optionally use threads to handle I/O, allowing to serve - 2 times as much operations per second in a single instance when - pipelining cannot be used. -* Diskless replication is now supported even on replicas: a replica is now - able, under certain conditions the user can configure, to load the RDB - in the first synchronization directly from the socket to the memory. -* Redis-benchmark now supports a Redis Cluster mode. -* SRANDMEMBER and similar commands have a better distribution. -* Redis-cli improvements. -* Systemd support rewritten. -* A Redis Cluster proxy was released here: - https://github.com/artix75/redis-cluster-proxy -* A Disque module for Redis was released here: - https://github.com/antirez/disque-module Thanks to all the users and developers who made this release possible. We'll follow up with more RC releases, until the code looks production ready and we don't get reports of serious issues for a while. -A special thank you for the amount of work put into this release -(in decreasing number of commits, only listing contributors with more -than a single commit) by: +A special thank you for the amount of work put into this release by: +- Oran Agra +- Yossi Gottlieb +- Viktor Söderqvist +- Yang Bodong +- Filipe Oliveira +- Guy Benoish +- Itamar Haber +- Madelyn Olson +- Wang Yuan +- Felipe Machado +- Wen Hui +- Tatsuya Arisawa +- Jonah H. Harris +- Raghav Muddur +- Jim Brunner +- Yaacov Hazan +- Allen Farris +- Chen Yang +- Nitai Caro +- sundb +- Meir Shpilraien +- maohuazhu +- Valentino Geron +- Zhao Zhao +- Qu Chen +- George Prekas +- Tyson Andre +- Uri Yagelnik +- Michael Grunder +- Huang Zw +- alexronke-channeladvisor +- Andy Pan +- Wu Yunlong +- Wei Kukey +- Yoav Steinberg +- Greg Femec +- Uri Shachar +- Nykolas Laurentino de Lima +- xhe +- zhenwei pi +- David CARLIER - 685 antirez - 81 zhaozhao.zz - 76 Oran Agra - 51 artix - 28 Madelyn Olson - 27 Yossi Gottlieb - 15 David Carlier - 14 Guy Benoish - 14 Guy Korland - 13 Itamar Haber - 9 Angus Pearson - 8 WuYunlong - 8 yongman - 7 vattezhang - 7 Chris Lamb - 5 Dvir Volk - 5 meir@redislabs.com - 5 chendianqiang - 5 John Sully - 4 dejun.xdj - 4 Daniel Dai - 4 Johannes Truschnigg - 4 swilly22 - 3 Bruce Merry - 3 filipecosta90 - 3 youjiali1995 - 2 James Rouzier - 2 Andrey Bugaevskiy - 2 Brad Solomon - 2 Hamid Alaei - 2 Michael Chaten - 2 Steve Webster - 2 Wander Hillen - 2 Weiliang Li - 2 Yuan Zhou - 2 charsyam - 2 hujie - 2 jem - 2 shenlongxing - 2 valentino - 2 zhudacai 00228490 - 2 喜欢兰花山丘 - -Migrating from 5.0 to 6.0 +Migrating from 6.0 to 6.2 ========================= -Redis 6.0 is mostly a strict superset of 5.0, you should not have any problem -upgrading your application from 5.0 to 6.0. However this is a list of small -non-backward compatible changes introduced in the 6.0 release: +Redis 6.2 is mostly a strict superset of 6.0, you should not have any problem +upgrading your application from 6.0 to 6.2. However there are some small changes +of behavior listed above, please make sure you are not badly affected by any of +them. -* The SPOP command no longer returns null when the set key does not - exist. Now it returns the empty set as it should and as happens when it is - called with a 0 argument. This is technically a fix, however it changes the - old behavior. +Specifically these sections: +* Command behavior changes +* Other behavior changes -------------------------------------------------------------------------------- -Credits: For each release, a list of changes with the relative author is -provided. Where not specified the implementation and design is done by -Salvatore Sanfilippo. Thanks to Redis Labs for making all this possible. -Also many thanks to all the other contributors and the amazing community -we have. - -Commit messages may contain additional credits. - -Enjoy, -Salvatore +Cheers, +The Redis team diff --git a/CONDUCT b/CONDUCT new file mode 100644 index 000000000..e0e15e268 --- /dev/null +++ b/CONDUCT @@ -0,0 +1,96 @@ +Contributor Covenant Code of Conduct +Our Pledge +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. +Our Standards +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, +and learning from the experience +* Focusing on what is best not just for us as individuals, but for the +overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or +advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others’ private information, such as a physical or email +address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a +professional setting + +Enforcement Responsibilities +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. +Scope +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. +Enforcement +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +this email address: redis@redis.io. +All complaints will be reviewed and investigated promptly and fairly. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. +Enforcement Guidelines +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: +1. Correction +Community Impact: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. +Consequence: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. +2. Warning +Community Impact: A violation through a single incident or series +of actions. +Consequence: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. +3. Temporary Ban +Community Impact: A serious violation of community standards, including +sustained inappropriate behavior. +Consequence: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. +4. Permanent Ban +Community Impact: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. +Consequence: A permanent ban from any sort of public interaction within +the community. +Attribution +This Code of Conduct is adapted from the Contributor Covenant, +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +Community Impact Guidelines were inspired by Mozilla’s code of conduct +enforcement ladder. +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/COPYING b/COPYING index 56e49f8c0..edc06c580 100644 --- a/COPYING +++ b/COPYING @@ -5,7 +5,8 @@ or contractors of EQ Alpha Technology are not licensed for use without express written permission of EQ Alpha Technology. All rights are reserved. Copyright (c) 2006-2020, Salvatore Sanfilippo -Copyright (C) 2019-2020, John Sully +Copyright (C) 2019-2021, John Sully +Copyright (C) 2020-2021, EQ Alpha Technology Ltd. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index bcb3e850e..9103efd88 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,11 @@ ![CI](https://github.com/JohnSully/KeyDB/workflows/CI/badge.svg?branch=unstable) [![StackShare](http://img.shields.io/badge/tech-stack-0690fa.svg?style=flat)](https://stackshare.io/eq-alpha-technology-inc/eq-alpha-technology-inc) -##### New! Want to extend KeyDB with Javascript? Try [ModJS](https://github.com/JohnSully/ModJS) +##### Want to extend KeyDB with Javascript? Try [ModJS](https://github.com/JohnSully/ModJS) ##### Need Help? Check out our extensive [documentation](https://docs.keydb.dev). -##### Have feedback? Take our quick survey: https://www.surveymonkey.com/r/Y9XNS93 - -##### KeyDB is Hiring! We are currently building out our dev team. If you are interested please see the posting here: https://keydb.dev/careers.html +##### NEW!!! KeyDB now has a Slack Community Workspace. Click [here](https://docs.keydb.dev/slack/) to learn more and join the KeyDB Community Slack workspace. What is KeyDB? -------------- @@ -124,6 +122,13 @@ installed): % ./runtest --tls +If TLS is built, running the tests with TLS enabled (you will need `tcl-tls` +installed): + + % ./utils/gen-test-certs.sh + % ./runtest --tls + + Fixing build problems with dependencies or cached build options --------- @@ -175,6 +180,18 @@ To compile against jemalloc on Mac OS X systems, use: % make MALLOC=jemalloc +Monotonic clock +--------------- + +By default, Redis will build using the POSIX clock_gettime function as the +monotonic clock source. On most modern systems, the internal processor clock +can be used to improve performance. Cautions can be found here: + http://oliveryang.net/2015/09/pitfalls-of-TSC-usage/ + +To build with support for the processor's internal instruction clock, use: + + % make CFLAGS="-DUSE_PROCESSOR_CLOCK" + Verbose build ------------- diff --git a/deps/Makefile b/deps/Makefile index c1c30ae54..d02882b1b 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -39,6 +39,7 @@ distclean: -(cd lua && $(MAKE) clean) > /dev/null || true -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true -(cd rocksdb && $(MAKE) clean) > /dev/null || true + -(cd hdr_histogram && $(MAKE) clean) > /dev/null || true -(rm -f .make-*) .PHONY: distclean @@ -64,18 +65,24 @@ memkind: cd memkind && $(MAKE) +hdr_histogram: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd hdr_histogram && $(MAKE) + +.PHONY: hdr_histogram + ifeq ($(uname_S),SunOS) # Make isinf() available LUA_CFLAGS= -D__C99FEATURES__=1 endif -LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS) +LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS) LUA_LDFLAGS+= $(LDFLAGS) # lua's Makefile defines AR="ar rcu", which is unusual, and makes it more # challenging to cross-compile lua (and redis). These defines make it easier # to fit redis into cross-compilation environments, which typically set AR. AR=ar -ARFLAGS=rcu +ARFLAGS=rc lua: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) @@ -88,7 +95,7 @@ JEMALLOC_LDFLAGS= $(LDFLAGS) jemalloc: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence --disable-cxx CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" + cd jemalloc && ./configure --with-version=5.2.1-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --disable-cxx CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a .PHONY: jemalloc diff --git a/deps/README.md b/deps/README.md index 02c99052f..c3ec424b6 100644 --- a/deps/README.md +++ b/deps/README.md @@ -17,7 +17,7 @@ active defragmentation logic. However this feature of Redis is not mandatory and Redis is able to understand if the Jemalloc version it is compiled against supports such Redis-specific modifications. So in theory, if you are not interested in the active defragmentation, you can replace Jemalloc -just following tose steps: +just following these steps: 1. Remove the jemalloc directory. 2. Substitute it with the new jemalloc source tree. diff --git a/deps/hdr_histogram/COPYING.txt b/deps/hdr_histogram/COPYING.txt new file mode 100644 index 000000000..0e259d42c --- /dev/null +++ b/deps/hdr_histogram/COPYING.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/deps/hdr_histogram/LICENSE.txt b/deps/hdr_histogram/LICENSE.txt new file mode 100644 index 000000000..9b4e66ed7 --- /dev/null +++ b/deps/hdr_histogram/LICENSE.txt @@ -0,0 +1,41 @@ +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/hdr_histogram/Makefile b/deps/hdr_histogram/Makefile new file mode 100644 index 000000000..83d3760ae --- /dev/null +++ b/deps/hdr_histogram/Makefile @@ -0,0 +1,20 @@ +STD= +WARN= -Wall +OPT= -Os + +R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) +R_LDFLAGS= $(LDFLAGS) +DEBUG= -g + +R_CC=$(CC) $(R_CFLAGS) +R_LD=$(CC) $(R_LDFLAGS) + +hdr_histogram.o: hdr_histogram.h hdr_histogram.c + +.c.o: + $(R_CC) -c $< + +clean: + rm -f *.o + + diff --git a/deps/hdr_histogram/README.md b/deps/hdr_histogram/README.md new file mode 100644 index 000000000..5f62c234c --- /dev/null +++ b/deps/hdr_histogram/README.md @@ -0,0 +1,10 @@ +HdrHistogram_c v0.11.0 + +---------------------------------------------- + +This port contains a subset of the 'C' version of High Dynamic Range (HDR) Histogram available at [github.com/HdrHistogram/HdrHistogram_c](https://github.com/HdrHistogram/HdrHistogram_c). + + +The code present on `hdr_histogram.c`, `hdr_histogram.h`, and `hdr_atomic.c` was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/. \ No newline at end of file diff --git a/deps/hdr_histogram/hdr_atomic.h b/deps/hdr_histogram/hdr_atomic.h new file mode 100644 index 000000000..ae1056a83 --- /dev/null +++ b/deps/hdr_histogram/hdr_atomic.h @@ -0,0 +1,146 @@ +/** + * hdr_atomic.h + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_ATOMIC_H__ +#define HDR_ATOMIC_H__ + + +#if defined(_MSC_VER) + +#include +#include +#include + +static void __inline * hdr_atomic_load_pointer(void** pointer) +{ + _ReadBarrier(); + return *pointer; +} + +static void hdr_atomic_store_pointer(void** pointer, void* value) +{ + _WriteBarrier(); + *pointer = value; +} + +static int64_t __inline hdr_atomic_load_64(int64_t* field) +{ + _ReadBarrier(); + return *field; +} + +static void __inline hdr_atomic_store_64(int64_t* field, int64_t value) +{ + _WriteBarrier(); + *field = value; +} + +static int64_t __inline hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchange64(field, value); +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, value, comparand); + } + while (comparand != initial_value); + + return initial_value; +#endif +} + +static int64_t __inline hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchangeAdd64(field, value) + value; +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, comparand + value, comparand); + } + while (comparand != initial_value); + + return initial_value + value; +#endif +} + +static bool __inline hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + return *expected == _InterlockedCompareExchange64(field, desired, *expected); +} + +#elif defined(__ATOMIC_SEQ_CST) + +#define hdr_atomic_load_pointer(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_pointer(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_load_64(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_64(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_exchange_64(f,i) __atomic_exchange_n(f,i, __ATOMIC_SEQ_CST) +#define hdr_atomic_add_fetch_64(field, value) __atomic_add_fetch(field, value, __ATOMIC_SEQ_CST) +#define hdr_atomic_compare_exchange_64(field, expected, desired) __atomic_compare_exchange_n(field, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) + +#elif defined(__x86_64__) + +#include +#include + +static inline void* hdr_atomic_load_pointer(void** pointer) +{ + void* p = *pointer; + asm volatile ("" ::: "memory"); + return p; +} + +static inline void hdr_atomic_store_pointer(void** pointer, void* value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*pointer)); +} + +static inline int64_t hdr_atomic_load_64(int64_t* field) +{ + int64_t i = *field; + asm volatile ("" ::: "memory"); + return i; +} + +static inline void hdr_atomic_store_64(int64_t* field, int64_t value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*field)); +} + +static inline int64_t hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ + int64_t result = 0; + asm volatile ("lock; xchgq %1, %2" : "=r" (result), "+q" (value), "+m" (*field)); + return result; +} + +static inline int64_t hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ + return __sync_add_and_fetch(field, value); +} + +static inline bool hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + int64_t original; + asm volatile( "lock; cmpxchgq %2, %1" : "=a"(original), "+m"(*field) : "q"(desired), "0"(*expected)); + return original == *expected; +} + +#else + +#error "Unable to determine atomic operations for your platform" + +#endif + +#endif /* HDR_ATOMIC_H__ */ diff --git a/deps/hdr_histogram/hdr_histogram.c b/deps/hdr_histogram/hdr_histogram.c new file mode 100644 index 000000000..5f5286f2f --- /dev/null +++ b/deps/hdr_histogram/hdr_histogram.c @@ -0,0 +1,1155 @@ +/** + * hdr_histogram.c + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdr_histogram.h" +#include "hdr_atomic.h" + +/* ###### ####### ## ## ## ## ######## ###### */ +/* ## ## ## ## ## ## ### ## ## ## ## */ +/* ## ## ## ## ## #### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ###### */ +/* ## ## ## ## ## ## #### ## ## */ +/* ## ## ## ## ## ## ## ### ## ## ## */ +/* ###### ####### ####### ## ## ## ###### */ + +static int32_t normalize_index(const struct hdr_histogram* h, int32_t index) +{ + int32_t normalized_index; + int32_t adjustment = 0; + if (h->normalizing_index_offset == 0) + { + return index; + } + + normalized_index = index - h->normalizing_index_offset; + + if (normalized_index < 0) + { + adjustment = h->counts_len; + } + else if (normalized_index >= h->counts_len) + { + adjustment = -h->counts_len; + } + + return normalized_index + adjustment; +} + +static int64_t counts_get_direct(const struct hdr_histogram* h, int32_t index) +{ + return h->counts[index]; +} + +static int64_t counts_get_normalised(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_direct(h, normalize_index(h, index)); +} + +static void counts_inc_normalised( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + h->counts[normalised_index] += value; + h->total_count += value; +} + +static void counts_inc_normalised_atomic( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + + hdr_atomic_add_fetch_64(&h->counts[normalised_index], value); + hdr_atomic_add_fetch_64(&h->total_count, value); +} + +static void update_min_max(struct hdr_histogram* h, int64_t value) +{ + h->min_value = (value < h->min_value && value != 0) ? value : h->min_value; + h->max_value = (value > h->max_value) ? value : h->max_value; +} + +static void update_min_max_atomic(struct hdr_histogram* h, int64_t value) +{ + int64_t current_min_value; + int64_t current_max_value; + do + { + current_min_value = hdr_atomic_load_64(&h->min_value); + + if (0 == value || current_min_value <= value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->min_value, ¤t_min_value, value)); + + do + { + current_max_value = hdr_atomic_load_64(&h->max_value); + + if (value <= current_max_value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->max_value, ¤t_max_value, value)); +} + + +/* ## ## ######## #### ## #### ######## ## ## */ +/* ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## #### */ +/* ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## */ +/* ####### ## #### ######## #### ## ## */ + +static int64_t power(int64_t base, int64_t exp) +{ + int64_t result = 1; + while(exp) + { + result *= base; exp--; + } + return result; +} + +#if defined(_MSC_VER) +# if defined(_WIN64) +# pragma intrinsic(_BitScanReverse64) +# else +# pragma intrinsic(_BitScanReverse) +# endif +#endif + +static int32_t count_leading_zeros_64(int64_t value) +{ +#if defined(_MSC_VER) + uint32_t leading_zero = 0; +#if defined(_WIN64) + _BitScanReverse64(&leading_zero, value); +#else + uint32_t high = value >> 32; + if (_BitScanReverse(&leading_zero, high)) + { + leading_zero += 32; + } + else + { + uint32_t low = value & 0x00000000FFFFFFFF; + _BitScanReverse(&leading_zero, low); + } +#endif + return 63 - leading_zero; /* smallest power of 2 containing value */ +#else + return __builtin_clzll(value); /* smallest power of 2 containing value */ +#endif +} + +static int32_t get_bucket_index(const struct hdr_histogram* h, int64_t value) +{ + int32_t pow2ceiling = 64 - count_leading_zeros_64(value | h->sub_bucket_mask); /* smallest power of 2 containing value */ + return pow2ceiling - h->unit_magnitude - (h->sub_bucket_half_count_magnitude + 1); +} + +static int32_t get_sub_bucket_index(int64_t value, int32_t bucket_index, int32_t unit_magnitude) +{ + return (int32_t)(value >> (bucket_index + unit_magnitude)); +} + +static int32_t counts_index(const struct hdr_histogram* h, int32_t bucket_index, int32_t sub_bucket_index) +{ + /* Calculate the index for the first entry in the bucket: */ + /* (The following is the equivalent of ((bucket_index + 1) * subBucketHalfCount) ): */ + int32_t bucket_base_index = (bucket_index + 1) << h->sub_bucket_half_count_magnitude; + /* Calculate the offset in the bucket: */ + int32_t offset_in_bucket = sub_bucket_index - h->sub_bucket_half_count; + /* The following is the equivalent of ((sub_bucket_index - subBucketHalfCount) + bucketBaseIndex; */ + return bucket_base_index + offset_in_bucket; +} + +static int64_t value_from_index(int32_t bucket_index, int32_t sub_bucket_index, int32_t unit_magnitude) +{ + return ((int64_t) sub_bucket_index) << (bucket_index + unit_magnitude); +} + +int32_t counts_index_for(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + + return counts_index(h, bucket_index, sub_bucket_index); +} + +int64_t hdr_value_at_index(const struct hdr_histogram *h, int32_t index) +{ + int32_t bucket_index = (index >> h->sub_bucket_half_count_magnitude) - 1; + int32_t sub_bucket_index = (index & (h->sub_bucket_half_count - 1)) + h->sub_bucket_half_count; + + if (bucket_index < 0) + { + sub_bucket_index -= h->sub_bucket_half_count; + bucket_index = 0; + } + + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + int32_t adjusted_bucket = (sub_bucket_index >= h->sub_bucket_count) ? (bucket_index + 1) : bucket_index; + return INT64_C(1) << (h->unit_magnitude + adjusted_bucket); +} + +static int64_t lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + hdr_size_of_equivalent_value_range(h, value); +} + +static int64_t highest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return hdr_next_non_equivalent_value(h, value) - 1; +} + +int64_t hdr_median_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + (hdr_size_of_equivalent_value_range(h, value) >> 1); +} + +static int64_t non_zero_min(const struct hdr_histogram* h) +{ + if (INT64_MAX == h->min_value) + { + return INT64_MAX; + } + + return lowest_equivalent_value(h, h->min_value); +} + +void hdr_reset_internal_counters(struct hdr_histogram* h) +{ + int min_non_zero_index = -1; + int max_index = -1; + int64_t observed_total_count = 0; + int i; + + for (i = 0; i < h->counts_len; i++) + { + int64_t count_at_index; + + if ((count_at_index = counts_get_direct(h, i)) > 0) + { + observed_total_count += count_at_index; + max_index = i; + if (min_non_zero_index == -1 && i != 0) + { + min_non_zero_index = i; + } + } + } + + if (max_index == -1) + { + h->max_value = 0; + } + else + { + int64_t max_value = hdr_value_at_index(h, max_index); + h->max_value = highest_equivalent_value(h, max_value); + } + + if (min_non_zero_index == -1) + { + h->min_value = INT64_MAX; + } + else + { + h->min_value = hdr_value_at_index(h, min_non_zero_index); + } + + h->total_count = observed_total_count; +} + +static int32_t buckets_needed_to_cover_value(int64_t value, int32_t sub_bucket_count, int32_t unit_magnitude) +{ + int64_t smallest_untrackable_value = ((int64_t) sub_bucket_count) << unit_magnitude; + int32_t buckets_needed = 1; + while (smallest_untrackable_value <= value) + { + if (smallest_untrackable_value > INT64_MAX / 2) + { + return buckets_needed + 1; + } + smallest_untrackable_value <<= 1; + buckets_needed++; + } + + return buckets_needed; +} + +/* ## ## ######## ## ## ####### ######## ## ## */ +/* ### ### ## ### ### ## ## ## ## ## ## */ +/* #### #### ## #### #### ## ## ## ## #### */ +/* ## ### ## ###### ## ### ## ## ## ######## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ## ## ####### ## ## ## */ + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg) +{ + int32_t sub_bucket_count_magnitude; + int64_t largest_value_with_single_unit_resolution; + + if (lowest_trackable_value < 1 || + significant_figures < 1 || 5 < significant_figures || + lowest_trackable_value * 2 > highest_trackable_value) + { + return EINVAL; + } + + cfg->lowest_trackable_value = lowest_trackable_value; + cfg->significant_figures = significant_figures; + cfg->highest_trackable_value = highest_trackable_value; + + largest_value_with_single_unit_resolution = 2 * power(10, significant_figures); + sub_bucket_count_magnitude = (int32_t) ceil(log((double)largest_value_with_single_unit_resolution) / log(2)); + cfg->sub_bucket_half_count_magnitude = ((sub_bucket_count_magnitude > 1) ? sub_bucket_count_magnitude : 1) - 1; + + cfg->unit_magnitude = (int32_t) floor(log((double)lowest_trackable_value) / log(2)); + + cfg->sub_bucket_count = (int32_t) pow(2, (cfg->sub_bucket_half_count_magnitude + 1)); + cfg->sub_bucket_half_count = cfg->sub_bucket_count / 2; + cfg->sub_bucket_mask = ((int64_t) cfg->sub_bucket_count - 1) << cfg->unit_magnitude; + + if (cfg->unit_magnitude + cfg->sub_bucket_half_count_magnitude > 61) + { + return EINVAL; + } + + cfg->bucket_count = buckets_needed_to_cover_value(highest_trackable_value, cfg->sub_bucket_count, (int32_t)cfg->unit_magnitude); + cfg->counts_len = (cfg->bucket_count + 1) * (cfg->sub_bucket_count / 2); + + return 0; +} + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg) +{ + h->lowest_trackable_value = cfg->lowest_trackable_value; + h->highest_trackable_value = cfg->highest_trackable_value; + h->unit_magnitude = (int32_t)cfg->unit_magnitude; + h->significant_figures = (int32_t)cfg->significant_figures; + h->sub_bucket_half_count_magnitude = cfg->sub_bucket_half_count_magnitude; + h->sub_bucket_half_count = cfg->sub_bucket_half_count; + h->sub_bucket_mask = cfg->sub_bucket_mask; + h->sub_bucket_count = cfg->sub_bucket_count; + h->min_value = INT64_MAX; + h->max_value = 0; + h->normalizing_index_offset = 0; + h->conversion_ratio = 1.0; + h->bucket_count = cfg->bucket_count; + h->counts_len = cfg->counts_len; + h->total_count = 0; +} + +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result) +{ + int64_t* counts; + struct hdr_histogram_bucket_config cfg; + struct hdr_histogram* histogram; + + int r = hdr_calculate_bucket_config(lowest_trackable_value, highest_trackable_value, significant_figures, &cfg); + if (r) + { + return r; + } + + counts = (int64_t*) calloc((size_t) cfg.counts_len, sizeof(int64_t)); + if (!counts) + { + return ENOMEM; + } + + histogram = (struct hdr_histogram*) calloc(1, sizeof(struct hdr_histogram)); + if (!histogram) + { + free(counts); + return ENOMEM; + } + + histogram->counts = counts; + + hdr_init_preallocated(histogram, &cfg); + *result = histogram; + + return 0; +} + +void hdr_close(struct hdr_histogram* h) +{ + if (h) { + free(h->counts); + free(h); + } +} + +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result) +{ + return hdr_init(1, highest_trackable_value, significant_figures, result); +} + +/* reset a histogram to zero. */ +void hdr_reset(struct hdr_histogram *h) +{ + h->total_count=0; + h->min_value = INT64_MAX; + h->max_value = 0; + memset(h->counts, 0, (sizeof(int64_t) * h->counts_len)); +} + +size_t hdr_get_memory_size(struct hdr_histogram *h) +{ + return sizeof(struct hdr_histogram) + h->counts_len * sizeof(int64_t); +} + +/* ## ## ######## ######## ### ######## ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ## ## ## ## ## ###### ###### */ +/* ## ## ## ## ## ######### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## */ +/* ####### ## ######## ## ## ## ######## ###### */ + + +bool hdr_record_value(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values(h, value, 1); +} + +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values_atomic(h, value, 1); +} + +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count) +{ + int32_t counts_index; + + if (value < 0) + { + return false; + } + + counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised(h, counts_index, count); + update_min_max(h, value); + + return true; +} + +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count) +{ + int32_t counts_index; + + if (value < 0) + { + return false; + } + + counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised_atomic(h, counts_index, count); + update_min_max_atomic(h, value); + + return true; +} + +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values(h, value, 1, expected_interval); +} + +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values_atomic(h, value, 1, expected_interval); +} + +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + int64_t missing_value; + + if (!hdr_record_values(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values(h, missing_value, count)) + { + return false; + } + } + + return true; +} + +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + int64_t missing_value; + + if (!hdr_record_values_atomic(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values_atomic(h, missing_value, count)) + { + return false; + } + } + + return true; +} + +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from) +{ + struct hdr_iter iter; + int64_t dropped = 0; + hdr_iter_recorded_init(&iter, from); + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_values(h, value, count)) + { + dropped += count; + } + } + + return dropped; +} + +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval) +{ + struct hdr_iter iter; + int64_t dropped = 0; + hdr_iter_recorded_init(&iter, from); + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_corrected_values(h, value, count, expected_interval)) + { + dropped += count; + } + } + + return dropped; +} + + + +/* ## ## ### ## ## ## ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ###### ###### */ +/* ## ## ######### ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ### ## ## ######## ####### ######## ###### */ + + +int64_t hdr_max(const struct hdr_histogram* h) +{ + if (0 == h->max_value) + { + return 0; + } + + return highest_equivalent_value(h, h->max_value); +} + +int64_t hdr_min(const struct hdr_histogram* h) +{ + if (0 < hdr_count_at_index(h, 0)) + { + return 0; + } + + return non_zero_min(h); +} + +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile) +{ + struct hdr_iter iter; + int64_t total = 0; + double requested_percentile = percentile < 100.0 ? percentile : 100.0; + int64_t count_at_percentile = + (int64_t) (((requested_percentile / 100) * h->total_count) + 0.5); + count_at_percentile = count_at_percentile > 1 ? count_at_percentile : 1; + + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + total += iter.count; + + if (total >= count_at_percentile) + { + int64_t value_from_index = iter.value; + return highest_equivalent_value(h, value_from_index); + } + } + + return 0; +} + +double hdr_mean(const struct hdr_histogram* h) +{ + struct hdr_iter iter; + int64_t total = 0; + + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + total += iter.count * hdr_median_equivalent_value(h, iter.value); + } + } + + return (total * 1.0) / h->total_count; +} + +double hdr_stddev(const struct hdr_histogram* h) +{ + double mean = hdr_mean(h); + double geometric_dev_total = 0.0; + + struct hdr_iter iter; + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + double dev = (hdr_median_equivalent_value(h, iter.value) * 1.0) - mean; + geometric_dev_total += (dev * dev) * iter.count; + } + } + + return sqrt(geometric_dev_total / h->total_count); +} + +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b) +{ + return lowest_equivalent_value(h, a) == lowest_equivalent_value(h, b); +} + +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return lowest_equivalent_value(h, value); +} + +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value) +{ + return counts_get_normalised(h, counts_index_for(h, value)); +} + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_normalised(h, index); +} + + +/* #### ######## ######## ######## ### ######## ####### ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ###### ######## ## ## ## ## ## ######## ###### */ +/* ## ## ## ## ## ######### ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* #### ## ######## ## ## ## ## ## ####### ## ## ###### */ + + +static bool has_buckets(struct hdr_iter* iter) +{ + return iter->counts_index < iter->h->counts_len; +} + +static bool has_next(struct hdr_iter* iter) +{ + return iter->cumulative_count < iter->total_count; +} + +static bool move_next(struct hdr_iter* iter) +{ + iter->counts_index++; + + if (!has_buckets(iter)) + { + return false; + } + + iter->count = counts_get_normalised(iter->h, iter->counts_index); + iter->cumulative_count += iter->count; + + iter->value = hdr_value_at_index(iter->h, iter->counts_index); + iter->highest_equivalent_value = highest_equivalent_value(iter->h, iter->value); + iter->lowest_equivalent_value = lowest_equivalent_value(iter->h, iter->value); + iter->median_equivalent_value = hdr_median_equivalent_value(iter->h, iter->value); + + return true; +} + +static int64_t peek_next_value_from_index(struct hdr_iter* iter) +{ + return hdr_value_at_index(iter->h, iter->counts_index + 1); +} + +static bool next_value_greater_than_reporting_level_upper_bound( + struct hdr_iter *iter, int64_t reporting_level_upper_bound) +{ + if (iter->counts_index >= iter->h->counts_len) + { + return false; + } + + return peek_next_value_from_index(iter) > reporting_level_upper_bound; +} + +static bool basic_iter_next(struct hdr_iter *iter) +{ + if (!has_next(iter) || iter->counts_index >= iter->h->counts_len) + { + return false; + } + + move_next(iter); + + return true; +} + +static void update_iterated_values(struct hdr_iter* iter, int64_t new_value_iterated_to) +{ + iter->value_iterated_from = iter->value_iterated_to; + iter->value_iterated_to = new_value_iterated_to; +} + +static bool all_values_iter_next(struct hdr_iter* iter) +{ + bool result = move_next(iter); + + if (result) + { + update_iterated_values(iter, iter->value); + } + + return result; +} + +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + iter->h = h; + + iter->counts_index = -1; + iter->total_count = h->total_count; + iter->count = 0; + iter->cumulative_count = 0; + iter->value = 0; + iter->highest_equivalent_value = 0; + iter->value_iterated_from = 0; + iter->value_iterated_to = 0; + + iter->_next_fp = all_values_iter_next; +} + +bool hdr_iter_next(struct hdr_iter* iter) +{ + return iter->_next_fp(iter); +} + +/* ######## ######## ######## ###### ######## ## ## ######## #### ## ######## ###### */ +/* ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## #### ## ## ## ## ## ## */ +/* ######## ###### ######## ## ###### ## ## ## ## ## ## ###### ###### */ +/* ## ## ## ## ## ## ## #### ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## */ +/* ## ######## ## ## ###### ######## ## ## ## #### ######## ######## ###### */ + +static bool percentile_iter_next(struct hdr_iter* iter) +{ + int64_t temp, half_distance, percentile_reporting_ticks; + + struct hdr_iter_percentiles* percentiles = &iter->specifics.percentiles; + + if (!has_next(iter)) + { + if (percentiles->seen_last_value) + { + return false; + } + + percentiles->seen_last_value = true; + percentiles->percentile = 100.0; + + return true; + } + + if (iter->counts_index == -1 && !basic_iter_next(iter)) + { + return false; + } + + do + { + double current_percentile = (100.0 * (double) iter->cumulative_count) / iter->h->total_count; + if (iter->count != 0 && + percentiles->percentile_to_iterate_to <= current_percentile) + { + update_iterated_values(iter, highest_equivalent_value(iter->h, iter->value)); + + percentiles->percentile = percentiles->percentile_to_iterate_to; + temp = (int64_t)(log(100 / (100.0 - (percentiles->percentile_to_iterate_to))) / log(2)) + 1; + half_distance = (int64_t) pow(2, (double) temp); + percentile_reporting_ticks = percentiles->ticks_per_half_distance * half_distance; + percentiles->percentile_to_iterate_to += 100.0 / percentile_reporting_ticks; + + return true; + } + } + while (basic_iter_next(iter)); + + return true; +} + +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance) +{ + iter->h = h; + + hdr_iter_init(iter, h); + + iter->specifics.percentiles.seen_last_value = false; + iter->specifics.percentiles.ticks_per_half_distance = ticks_per_half_distance; + iter->specifics.percentiles.percentile_to_iterate_to = 0.0; + iter->specifics.percentiles.percentile = 0.0; + + iter->_next_fp = percentile_iter_next; +} + +static void format_line_string(char* str, size_t len, int significant_figures, format_type format) +{ +#if defined(_MSC_VER) +#define snprintf _snprintf +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + const char* format_str = "%s%d%s"; + + switch (format) + { + case CSV: + snprintf(str, len, format_str, "%.", significant_figures, "f,%f,%d,%.2f\n"); + break; + case CLASSIC: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + break; + default: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + } +#if defined(_MSC_VER) +#undef snprintf +#pragma warning(pop) +#endif +} + + +/* ######## ######## ###### ####### ######## ######## ######## ######## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ######## ###### ## ## ## ######## ## ## ###### ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ###### ####### ## ## ######## ######## ######## */ + + +static bool recorded_iter_next(struct hdr_iter* iter) +{ + while (basic_iter_next(iter)) + { + if (iter->count != 0) + { + update_iterated_values(iter, iter->value); + + iter->specifics.recorded.count_added_in_this_iteration_step = iter->count; + return true; + } + } + + return false; +} + +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + hdr_iter_init(iter, h); + + iter->specifics.recorded.count_added_in_this_iteration_step = 0; + + iter->_next_fp = recorded_iter_next; +} + +/* ## #### ## ## ######## ### ######## */ +/* ## ## ### ## ## ## ## ## ## */ +/* ## ## #### ## ## ## ## ## ## */ +/* ## ## ## ## ## ###### ## ## ######## */ +/* ## ## ## #### ## ######### ## ## */ +/* ## ## ## ### ## ## ## ## ## */ +/* ######## #### ## ## ######## ## ## ## ## */ + + +static bool iter_linear_next(struct hdr_iter* iter) +{ + struct hdr_iter_linear* linear = &iter->specifics.linear; + + linear->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, linear->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= linear->next_value_reporting_level_lowest_equivalent) + { + update_iterated_values(iter, linear->next_value_reporting_level); + + linear->next_value_reporting_level += linear->value_units_per_bucket; + linear->next_value_reporting_level_lowest_equivalent = + lowest_equivalent_value(iter->h, linear->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + linear->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + + +void hdr_iter_linear_init(struct hdr_iter* iter, const struct hdr_histogram* h, int64_t value_units_per_bucket) +{ + hdr_iter_init(iter, h); + + iter->specifics.linear.count_added_in_this_iteration_step = 0; + iter->specifics.linear.value_units_per_bucket = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_per_bucket); + + iter->_next_fp = iter_linear_next; +} + +void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket) +{ + iter->specifics.linear.value_units_per_bucket = value_units_per_bucket; +} + +/* ## ####### ###### ### ######## #### ######## ## ## ## ## #### ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ### ### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## */ +/* ## ## ## ## #### ## ## ######## ## ## ######### ## ### ## ## ## */ +/* ## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ######## ####### ###### ## ## ## ## #### ## ## ## ## ## #### ###### */ + +static bool log_iter_next(struct hdr_iter *iter) +{ + struct hdr_iter_log* logarithmic = &iter->specifics.log; + + logarithmic->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, logarithmic->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= logarithmic->next_value_reporting_level_lowest_equivalent) + { + update_iterated_values(iter, logarithmic->next_value_reporting_level); + + logarithmic->next_value_reporting_level *= (int64_t)logarithmic->log_base; + logarithmic->next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(iter->h, logarithmic->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + logarithmic->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base) +{ + hdr_iter_init(iter, h); + iter->specifics.log.count_added_in_this_iteration_step = 0; + iter->specifics.log.log_base = log_base; + iter->specifics.log.next_value_reporting_level = value_units_first_bucket; + iter->specifics.log.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_first_bucket); + + iter->_next_fp = log_iter_next; +} + +/* Printing. */ + +static const char* format_head_string(format_type format) +{ + switch (format) + { + case CSV: + return "%s,%s,%s,%s\n"; + case CLASSIC: + default: + return "%12s %12s %12s %12s\n\n"; + } +} + +static const char CLASSIC_FOOTER[] = + "#[Mean = %12.3f, StdDeviation = %12.3f]\n" + "#[Max = %12.3f, Total count = %12" PRIu64 "]\n" + "#[Buckets = %12d, SubBuckets = %12d]\n"; + +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format) +{ + char line_format[25]; + const char* head_format; + int rc = 0; + struct hdr_iter iter; + struct hdr_iter_percentiles * percentiles; + + format_line_string(line_format, 25, h->significant_figures, format); + head_format = format_head_string(format); + + hdr_iter_percentile_init(&iter, h, ticks_per_half_distance); + + if (fprintf( + stream, head_format, + "Value", "Percentile", "TotalCount", "1/(1-Percentile)") < 0) + { + rc = EIO; + goto cleanup; + } + + percentiles = &iter.specifics.percentiles; + while (hdr_iter_next(&iter)) + { + double value = iter.highest_equivalent_value / value_scale; + double percentile = percentiles->percentile / 100.0; + int64_t total_count = iter.cumulative_count; + double inverted_percentile = (1.0 / (1.0 - percentile)); + + if (fprintf( + stream, line_format, value, percentile, total_count, inverted_percentile) < 0) + { + rc = EIO; + goto cleanup; + } + } + + if (CLASSIC == format) + { + double mean = hdr_mean(h) / value_scale; + double stddev = hdr_stddev(h) / value_scale; + double max = hdr_max(h) / value_scale; + + if (fprintf( + stream, CLASSIC_FOOTER, mean, stddev, max, + h->total_count, h->bucket_count, h->sub_bucket_count) < 0) + { + rc = EIO; + goto cleanup; + } + } + + cleanup: + return rc; +} diff --git a/deps/hdr_histogram/hdr_histogram.h b/deps/hdr_histogram/hdr_histogram.h new file mode 100644 index 000000000..c26c56b20 --- /dev/null +++ b/deps/hdr_histogram/hdr_histogram.h @@ -0,0 +1,509 @@ +/** + * hdr_histogram.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + * + * The source for the hdr_histogram utilises a few C99 constructs, specifically + * the use of stdint/stdbool and inline variable declaration. + */ + +#ifndef HDR_HISTOGRAM_H +#define HDR_HISTOGRAM_H 1 + +#include +#include +#include + +struct hdr_histogram +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int32_t unit_magnitude; + int32_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int64_t min_value; + int64_t max_value; + int32_t normalizing_index_offset; + double conversion_ratio; + int32_t counts_len; + int64_t total_count; + int64_t* counts; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Allocate the memory and initialise the hdr_histogram. + * + * Due to the size of the histogram being the result of some reasonably + * involved math on the input parameters this function it is tricky to stack allocate. + * The histogram should be released with hdr_close + * + * @param lowest_trackable_value The smallest possible value to be put into the + * histogram. + * @param highest_trackable_value The largest possible value to be put into the + * histogram. + * @param significant_figures The level of precision for this histogram, i.e. the number + * of figures in a decimal number that will be maintained. E.g. a value of 3 will mean + * the results from the histogram will be accurate up to the first three digits. Must + * be a value between 1 and 5 (inclusive). + * @param result Output parameter to capture allocated histogram. + * @return 0 on success, EINVAL if lowest_trackable_value is < 1 or the + * significant_figure value is outside of the allowed range, ENOMEM if malloc + * failed. + */ +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result); + +/** + * Free the memory and close the hdr_histogram. + * + * @param h The histogram you want to close. + */ +void hdr_close(struct hdr_histogram* h); + +/** + * Allocate the memory and initialise the hdr_histogram. This is the equivalent of calling + * hdr_init(1, highest_trackable_value, significant_figures, result); + * + * @deprecated use hdr_init. + */ +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result); + + +/** + * Reset a histogram to zero - empty out a histogram and re-initialise it + * + * If you want to re-use an existing histogram, but reset everything back to zero, this + * is the routine to use. + * + * @param h The histogram you want to reset to empty. + * + */ +void hdr_reset(struct hdr_histogram* h); + +/** + * Get the memory size of the hdr_histogram. + * + * @param h "This" pointer + * @return The amount of memory used by the hdr_histogram in bytes + */ +size_t hdr_get_memory_size(struct hdr_histogram* h); + +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value(struct hdr_histogram* h, int64_t value); + +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value); + +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count); + +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count); + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval); + +/** + * Get minimum value from the histogram. Will return 2^63-1 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_min(const struct hdr_histogram* h); + +/** + * Get maximum value from the histogram. Will return 0 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_max(const struct hdr_histogram* h); + +/** + * Get the value at a specific percentile. + * + * @param h "This" pointer. + * @param percentile The percentile to get the value for + */ +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile); + +/** + * Gets the standard deviation for the values in the histogram. + * + * @param h "This" pointer + * @return The standard deviation + */ +double hdr_stddev(const struct hdr_histogram* h); + +/** + * Gets the mean for the values in the histogram. + * + * @param h "This" pointer + * @return The mean + */ +double hdr_mean(const struct hdr_histogram* h); + +/** + * Determine if two values are equivalent with the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param a first value to compare + * @param b second value to compare + * @return 'true' if values are equivalent with the histogram's resolution. + */ +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b); + +/** + * Get the lowest value that is equivalent to the given value within the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param value The given value + * @return The lowest value that is equivalent to the given value within the histogram's resolution. + */ +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Get the count of recorded values at a specific value + * (to within the histogram resolution at the value level). + * + * @param h "This" pointer + * @param value The value for which to provide the recorded count + * @return The total count of values recorded in the histogram within the value range that is + * {@literal >=} lowestEquivalentValue(value) and {@literal <=} highestEquivalentValue(value) + */ +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index); + +int64_t hdr_value_at_index(const struct hdr_histogram* h, int32_t index); + +struct hdr_iter_percentiles +{ + bool seen_last_value; + int32_t ticks_per_half_distance; + double percentile_to_iterate_to; + double percentile; +}; + +struct hdr_iter_recorded +{ + int64_t count_added_in_this_iteration_step; +}; + +struct hdr_iter_linear +{ + int64_t value_units_per_bucket; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +struct hdr_iter_log +{ + double log_base; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +/** + * The basic iterator. This is a generic structure + * that supports all of the types of iteration. Use + * the appropriate initialiser to get the desired + * iteration. + * + * @ + */ +struct hdr_iter +{ + const struct hdr_histogram* h; + /** raw index into the counts array */ + int32_t counts_index; + /** snapshot of the length at the time the iterator is created */ + int64_t total_count; + /** value directly from array for the current counts_index */ + int64_t count; + /** sum of all of the counts up to and including the count at this index */ + int64_t cumulative_count; + /** The current value based on counts_index */ + int64_t value; + int64_t highest_equivalent_value; + int64_t lowest_equivalent_value; + int64_t median_equivalent_value; + int64_t value_iterated_from; + int64_t value_iterated_to; + + union + { + struct hdr_iter_percentiles percentiles; + struct hdr_iter_recorded recorded; + struct hdr_iter_linear linear; + struct hdr_iter_log log; + } specifics; + + bool (* _next_fp)(struct hdr_iter* iter); + +}; + +/** + * Initalises the basic iterator. + * + * @param itr 'This' pointer + * @param h The histogram to iterate over + */ +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with percentiles. + */ +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance); + +/** + * Initialise the iterator for use with recorded values. + */ +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with linear values. + */ +void hdr_iter_linear_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_per_bucket); + +/** + * Update the iterator value units per bucket + */ +void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket); + +/** + * Initialise the iterator for use with logarithmic values + */ +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base); + +/** + * Iterate to the next value for the iterator. If there are no more values + * available return faluse. + * + * @param itr 'This' pointer + * @return 'false' if there are no values remaining for this iterator. + */ +bool hdr_iter_next(struct hdr_iter* iter); + +typedef enum +{ + CLASSIC, + CSV +} format_type; + +/** + * Print out a percentile based histogram to the supplied stream. Note that + * this call will not flush the FILE, this is left up to the user. + * + * @param h 'This' pointer + * @param stream The FILE to write the output to + * @param ticks_per_half_distance The number of iteration steps per half-distance to 100% + * @param value_scale Scale the output values by this amount + * @param format_type Format to use, e.g. CSV. + * @return 0 on success, error code on failure. EIO if an error occurs writing + * the output. + */ +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format); + +/** +* Internal allocation methods, used by hdr_dbl_histogram. +*/ +struct hdr_histogram_bucket_config +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int64_t unit_magnitude; + int64_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int32_t counts_len; +}; + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg); + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg); + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_median_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Used to reset counters after importing data manuallying into the histogram, used by the logging code + * and other custom serialisation tools. + */ +void hdr_reset_internal_counters(struct hdr_histogram* h); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index 8e50b5434..056959ffe 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -6,3 +6,4 @@ /*.a /*.pc *.dSYM +tags diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index dd8e0e73d..f9a9460ff 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -1,5 +1,4 @@ language: c -sudo: false compiler: - gcc - clang @@ -8,17 +7,34 @@ os: - linux - osx +dist: bionic + branches: only: - staging - trying - master + - /^release\/.*$/ + +install: + - if [ "$BITS" == "64" ]; then + wget https://github.com/redis/redis/archive/6.0.6.tar.gz; + tar -xzvf 6.0.6.tar.gz; + pushd redis-6.0.6 && BUILD_TLS=yes make && export PATH=$PWD/src:$PATH && popd; + fi before_script: - - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update; brew install redis; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ]; then + curl -O https://distfiles.macports.org/MacPorts/MacPorts-2.6.2-10.13-HighSierra.pkg; + sudo installer -pkg MacPorts-2.6.2-10.13-HighSierra.pkg -target /; + export PATH=$PATH:/opt/local/bin && sudo port -v selfupdate; + sudo port -N install openssl redis; + fi; addons: apt: + sources: + - sourceline: 'ppa:chris-lea/redis-server' packages: - libc6-dbg - libc6-dev @@ -27,14 +43,20 @@ addons: - libc6-dbg:i386 - gcc-multilib - g++-multilib + - libssl-dev + - libssl-dev:i386 - valgrind + - redis env: - BITS="32" - BITS="64" script: - - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON"; + if [ "$BITS" == "64" ]; then + EXTRA_CMAKE_OPTS="$EXTRA_CMAKE_OPTS -DENABLE_SSL_TESTS:BOOL=ON"; + fi; if [ "$TRAVIS_OS_NAME" == "osx" ]; then if [ "$BITS" == "32" ]; then CFLAGS="-m32 -Werror"; @@ -58,12 +80,24 @@ script: fi; fi; export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - make && make clean; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "64" ]; then + OPENSSL_PREFIX="$(ls -d /usr/local/Cellar/openssl@1.1/*)" USE_SSL=1 make; + fi; + else + USE_SSL=1 make; + fi; - mkdir build/ && cd build/ - cmake .. ${EXTRA_CMAKE_OPTS} - make VERBOSE=1 - - ctest -V + - if [ "$BITS" == "64" ]; then + TEST_SSL=1 SKIPS_AS_FAILS=1 ctest -V; + else + SKIPS_AS_FAILS=1 ctest -V; + fi; -matrix: +jobs: include: # Windows MinGW cross compile on Linux - os: linux @@ -89,9 +123,9 @@ matrix: - eval "${MATRIX_EVAL}" install: - choco install ninja + - choco install -y memurai-developer script: - mkdir build && cd build - - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && - ninja -v' - - ctest -V + - cmd.exe //C 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat' amd64 '&&' + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON '&&' ninja -v + - ./hiredis-test.exe diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index d1d37e515..271f1fcf3 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -1,28 +1,175 @@ -### 1.0.0 (unreleased) +## [1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) - (2020-08-03) + +Announcing Hiredis v1.0.0, which adds support for RESP3, SSL connections, allocator injection, and better Windows support! :tada: + +_A big thanks to everyone who helped with this release. The following list includes everyone who contributed at least five lines, sorted by lines contributed._ :sparkling_heart: + +[Michael Grunder](https://github.com/michael-grunder), [Yossi Gottlieb](https://github.com/yossigo), +[Mark Nunberg](https://github.com/mnunberg), [Marcus Geelnard](https://github.com/mbitsnbites), +[Justin Brewer](https://github.com/justinbrewer), [Valentino Geron](https://github.com/valentinogeron), +[Minun Dragonation](https://github.com/dragonation), [Omri Steiner](https://github.com/OmriSteiner), +[Sangmoon Yi](https://github.com/jman-krafton), [Jinjiazh](https://github.com/jinjiazhang), +[Odin Hultgren Van Der Horst](https://github.com/Miniwoffer), [Muhammad Zahalqa](https://github.com/tryfinally), +[Nick Rivera](https://github.com/heronr), [Qi Yang](https://github.com/movebean), +[kevin1018](https://github.com/kevin1018) + +[Full Changelog](https://github.com/redis/hiredis/compare/v0.14.1...v1.0.0) **BREAKING CHANGES**: -* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now - protocol errors. This is consistent with the RESP specification. On 32-bit - platforms, the upper bound is lowered to `SIZE_MAX`. +* `redisOptions` now has two timeout fields. One for connecting, and one for commands. If you're presently using `options->timeout` you will need to change it to use `options->connect_timeout`. (See [example](https://github.com/redis/hiredis/commit/38b5ae543f5c99eb4ccabbe277770fc6bc81226f#diff-86ba39d37aa829c8c82624cce4f049fbL36)) -* Change `redisReply.len` to `size_t`, as it denotes the the size of a string - - User code should compare this to `size_t` values as well. If it was used to - compare to other values, casting might be necessary or can be removed, if - casting was applied before. - -### 0.x.x (unreleased) -**BREAKING CHANGES**: - -* Change `redisReply.len` to `size_t`, as it denotes the the size of a string - -User code should compare this to `size_t` values as well. -If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now protocol errors. This is consistent + with the RESP specification. On 32-bit platforms, the upper bound is lowered to `SIZE_MAX`. * `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. +**New features:** +- Support for RESP3 + [\#697](https://github.com/redis/hiredis/pull/697), + [\#805](https://github.com/redis/hiredis/pull/805), + [\#819](https://github.com/redis/hiredis/pull/819), + [\#841](https://github.com/redis/hiredis/pull/841) + ([Yossi Gottlieb](https://github.com/yossigo), [Michael Grunder](https://github.com/michael-grunder)) +- Support for SSL connections + [\#645](https://github.com/redis/hiredis/pull/645), + [\#699](https://github.com/redis/hiredis/pull/699), + [\#702](https://github.com/redis/hiredis/pull/702), + [\#708](https://github.com/redis/hiredis/pull/708), + [\#711](https://github.com/redis/hiredis/pull/711), + [\#821](https://github.com/redis/hiredis/pull/821), + [more](https://github.com/redis/hiredis/pulls?q=is%3Apr+is%3Amerged+SSL) + ([Mark Nunberg](https://github.com/mnunberg), [Yossi Gottlieb](https://github.com/yossigo)) +- Run-time allocator injection + [\#800](https://github.com/redis/hiredis/pull/800) + ([Michael Grunder](https://github.com/michael-grunder)) +- Improved Windows support (including MinGW and Windows CI) + [\#652](https://github.com/redis/hiredis/pull/652), + [\#663](https://github.com/redis/hiredis/pull/663) + ([Marcus Geelnard](https://www.bitsnbites.eu/author/m/)) +- Adds support for distinct connect and command timeouts + [\#839](https://github.com/redis/hiredis/pull/839), + [\#829](https://github.com/redis/hiredis/pull/829) + ([Valentino Geron](https://github.com/valentinogeron)) +- Add generic pointer and destructor to `redisContext` that users can use for context. + [\#855](https://github.com/redis/hiredis/pull/855) + ([Michael Grunder](https://github.com/michael-grunder)) + +**Closed issues (that involved code changes):** + +- Makefile does not install TLS libraries [\#809](https://github.com/redis/hiredis/issues/809) +- redisConnectWithOptions should not set command timeout [\#722](https://github.com/redis/hiredis/issues/722), [\#829](https://github.com/redis/hiredis/pull/829) ([valentinogeron](https://github.com/valentinogeron)) +- Fix integer overflow in `sdsrange` [\#827](https://github.com/redis/hiredis/issues/827) +- INFO & CLUSTER commands failed when using RESP3 [\#802](https://github.com/redis/hiredis/issues/802) +- Windows compatibility patches [\#687](https://github.com/redis/hiredis/issues/687), [\#838](https://github.com/redis/hiredis/issues/838), [\#842](https://github.com/redis/hiredis/issues/842) +- RESP3 PUSH messages incorrectly use pending callback [\#825](https://github.com/redis/hiredis/issues/825) +- Asynchronous PSUBSCRIBE command fails when using RESP3 [\#815](https://github.com/redis/hiredis/issues/815) +- New SSL API [\#804](https://github.com/redis/hiredis/issues/804), [\#813](https://github.com/redis/hiredis/issues/813) +- Hard-coded limit of nested reply depth [\#794](https://github.com/redis/hiredis/issues/794) +- Fix TCP_NODELAY in Windows/OSX [\#679](https://github.com/redis/hiredis/issues/679), [\#690](https://github.com/redis/hiredis/issues/690), [\#779](https://github.com/redis/hiredis/issues/779), [\#785](https://github.com/redis/hiredis/issues/785), +- Added timers to libev adapter. [\#778](https://github.com/redis/hiredis/issues/778), [\#795](https://github.com/redis/hiredis/pull/795) +- Initialization discards const qualifier [\#777](https://github.com/redis/hiredis/issues/777) +- \[BUG\]\[MinGW64\] Error setting socket timeout [\#775](https://github.com/redis/hiredis/issues/775) +- undefined reference to hi_malloc [\#769](https://github.com/redis/hiredis/issues/769) +- hiredis pkg-config file incorrectly ignores multiarch libdir spec'n [\#767](https://github.com/redis/hiredis/issues/767) +- Don't use -G to build shared object on Solaris [\#757](https://github.com/redis/hiredis/issues/757) +- error when make USE\_SSL=1 [\#748](https://github.com/redis/hiredis/issues/748) +- Allow to change SSL Mode [\#646](https://github.com/redis/hiredis/issues/646) +- hiredis/adapters/libevent.h memleak [\#618](https://github.com/redis/hiredis/issues/618) +- redisLibuvPoll crash when server closes the connetion [\#545](https://github.com/redis/hiredis/issues/545) +- about redisAsyncDisconnect question [\#518](https://github.com/redis/hiredis/issues/518) +- hiredis adapters libuv error for help [\#508](https://github.com/redis/hiredis/issues/508) +- API/ABI changes analysis [\#506](https://github.com/redis/hiredis/issues/506) +- Memory leak patch in Redis [\#502](https://github.com/redis/hiredis/issues/502) +- Remove the depth limitation [\#421](https://github.com/redis/hiredis/issues/421) + +**Merged pull requests:** + +- Move SSL management to a distinct private pointer [\#855](https://github.com/redis/hiredis/pull/855) ([michael-grunder](https://github.com/michael-grunder)) +- Move include to sockcompat.h to maintain style [\#850](https://github.com/redis/hiredis/pull/850) ([michael-grunder](https://github.com/michael-grunder)) +- Remove erroneous tag and add license to push example [\#849](https://github.com/redis/hiredis/pull/849) ([michael-grunder](https://github.com/michael-grunder)) +- fix windows compiling with mingw [\#848](https://github.com/redis/hiredis/pull/848) ([rmalizia44](https://github.com/rmalizia44)) +- Some Windows quality of life improvements. [\#846](https://github.com/redis/hiredis/pull/846) ([michael-grunder](https://github.com/michael-grunder)) +- Use \_WIN32 define instead of WIN32 [\#845](https://github.com/redis/hiredis/pull/845) ([michael-grunder](https://github.com/michael-grunder)) +- Non Linux CI fixes [\#844](https://github.com/redis/hiredis/pull/844) ([michael-grunder](https://github.com/michael-grunder)) +- Resp3 oob push support [\#841](https://github.com/redis/hiredis/pull/841) ([michael-grunder](https://github.com/michael-grunder)) +- fix \#785: defer TCP\_NODELAY in async tcp connections [\#836](https://github.com/redis/hiredis/pull/836) ([OmriSteiner](https://github.com/OmriSteiner)) +- sdsrange overflow fix [\#830](https://github.com/redis/hiredis/pull/830) ([michael-grunder](https://github.com/michael-grunder)) +- Use explicit pointer casting for c++ compatibility [\#826](https://github.com/redis/hiredis/pull/826) ([aureus1](https://github.com/aureus1)) +- Document allocator injection and completeness fix in test.c [\#824](https://github.com/redis/hiredis/pull/824) ([michael-grunder](https://github.com/michael-grunder)) +- Use unique names for allocator struct members [\#823](https://github.com/redis/hiredis/pull/823) ([michael-grunder](https://github.com/michael-grunder)) +- New SSL API to replace redisSecureConnection\(\). [\#821](https://github.com/redis/hiredis/pull/821) ([yossigo](https://github.com/yossigo)) +- Add logic to handle RESP3 push messages [\#819](https://github.com/redis/hiredis/pull/819) ([michael-grunder](https://github.com/michael-grunder)) +- Use standrad isxdigit instead of custom helper function. [\#814](https://github.com/redis/hiredis/pull/814) ([tryfinally](https://github.com/tryfinally)) +- Fix missing SSL build/install options. [\#812](https://github.com/redis/hiredis/pull/812) ([yossigo](https://github.com/yossigo)) +- Add link to ABI tracker [\#808](https://github.com/redis/hiredis/pull/808) ([michael-grunder](https://github.com/michael-grunder)) +- Resp3 verbatim string support [\#805](https://github.com/redis/hiredis/pull/805) ([michael-grunder](https://github.com/michael-grunder)) +- Allow users to replace allocator and handle OOM everywhere. [\#800](https://github.com/redis/hiredis/pull/800) ([michael-grunder](https://github.com/michael-grunder)) +- Remove nested depth limitation. [\#797](https://github.com/redis/hiredis/pull/797) ([michael-grunder](https://github.com/michael-grunder)) +- Attempt to fix compilation on Solaris [\#796](https://github.com/redis/hiredis/pull/796) ([michael-grunder](https://github.com/michael-grunder)) +- Support timeouts in libev adapater [\#795](https://github.com/redis/hiredis/pull/795) ([michael-grunder](https://github.com/michael-grunder)) +- Fix pkgconfig when installing to a custom lib dir [\#793](https://github.com/redis/hiredis/pull/793) ([michael-grunder](https://github.com/michael-grunder)) +- Fix USE\_SSL=1 make/cmake on OSX and CMake tests [\#789](https://github.com/redis/hiredis/pull/789) ([michael-grunder](https://github.com/michael-grunder)) +- Use correct libuv call on Windows [\#784](https://github.com/redis/hiredis/pull/784) ([michael-grunder](https://github.com/michael-grunder)) +- Added CMake package config and fixed hiredis\_ssl on Windows [\#783](https://github.com/redis/hiredis/pull/783) ([michael-grunder](https://github.com/michael-grunder)) +- CMake: Set hiredis\_ssl shared object version. [\#780](https://github.com/redis/hiredis/pull/780) ([yossigo](https://github.com/yossigo)) +- Win32 tests and timeout fix [\#776](https://github.com/redis/hiredis/pull/776) ([michael-grunder](https://github.com/michael-grunder)) +- Provides an optional cleanup callback for async data. [\#768](https://github.com/redis/hiredis/pull/768) ([heronr](https://github.com/heronr)) +- Housekeeping fixes [\#764](https://github.com/redis/hiredis/pull/764) ([michael-grunder](https://github.com/michael-grunder)) +- install alloc.h [\#756](https://github.com/redis/hiredis/pull/756) ([ch1aki](https://github.com/ch1aki)) +- fix spelling mistakes [\#746](https://github.com/redis/hiredis/pull/746) ([ShooterIT](https://github.com/ShooterIT)) +- Free the reply in redisGetReply when passed NULL [\#741](https://github.com/redis/hiredis/pull/741) ([michael-grunder](https://github.com/michael-grunder)) +- Fix dead code in sslLogCallback relating to should\_log variable. [\#737](https://github.com/redis/hiredis/pull/737) ([natoscott](https://github.com/natoscott)) +- Fix typo in dict.c. [\#731](https://github.com/redis/hiredis/pull/731) ([Kevin-Xi](https://github.com/Kevin-Xi)) +- Adding an option to DISABLE\_TESTS [\#727](https://github.com/redis/hiredis/pull/727) ([pbotros](https://github.com/pbotros)) +- Update README with SSL support. [\#720](https://github.com/redis/hiredis/pull/720) ([yossigo](https://github.com/yossigo)) +- Fixes leaks in unit tests [\#715](https://github.com/redis/hiredis/pull/715) ([michael-grunder](https://github.com/michael-grunder)) +- SSL Tests [\#711](https://github.com/redis/hiredis/pull/711) ([yossigo](https://github.com/yossigo)) +- SSL Reorganization [\#708](https://github.com/redis/hiredis/pull/708) ([yossigo](https://github.com/yossigo)) +- Fix MSVC build. [\#706](https://github.com/redis/hiredis/pull/706) ([yossigo](https://github.com/yossigo)) +- SSL: Properly report SSL\_connect\(\) errors. [\#702](https://github.com/redis/hiredis/pull/702) ([yossigo](https://github.com/yossigo)) +- Silent SSL trace to stdout by default. [\#699](https://github.com/redis/hiredis/pull/699) ([yossigo](https://github.com/yossigo)) +- Port RESP3 support from Redis. [\#697](https://github.com/redis/hiredis/pull/697) ([yossigo](https://github.com/yossigo)) +- Removed whitespace before newline [\#691](https://github.com/redis/hiredis/pull/691) ([Miniwoffer](https://github.com/Miniwoffer)) +- Add install adapters header files [\#688](https://github.com/redis/hiredis/pull/688) ([kevin1018](https://github.com/kevin1018)) +- Remove unnecessary null check before free [\#684](https://github.com/redis/hiredis/pull/684) ([qlyoung](https://github.com/qlyoung)) +- redisReaderGetReply leak memory [\#671](https://github.com/redis/hiredis/pull/671) ([movebean](https://github.com/movebean)) +- fix timeout code in windows [\#670](https://github.com/redis/hiredis/pull/670) ([jman-krafton](https://github.com/jman-krafton)) +- test: fix errstr matching for musl libc [\#665](https://github.com/redis/hiredis/pull/665) ([ghost](https://github.com/ghost)) +- Windows: MinGW fixes and Windows Travis builders [\#663](https://github.com/redis/hiredis/pull/663) ([mbitsnbites](https://github.com/mbitsnbites)) +- The setsockopt and getsockopt API diffs from BSD socket and WSA one [\#662](https://github.com/redis/hiredis/pull/662) ([dragonation](https://github.com/dragonation)) +- Fix Compile Error On Windows \(Visual Studio\) [\#658](https://github.com/redis/hiredis/pull/658) ([jinjiazhang](https://github.com/jinjiazhang)) +- Fix NXDOMAIN test case [\#653](https://github.com/redis/hiredis/pull/653) ([michael-grunder](https://github.com/michael-grunder)) +- Add MinGW support [\#652](https://github.com/redis/hiredis/pull/652) ([mbitsnbites](https://github.com/mbitsnbites)) +- SSL Support [\#645](https://github.com/redis/hiredis/pull/645) ([mnunberg](https://github.com/mnunberg)) +- Fix Invalid argument after redisAsyncConnectUnix [\#644](https://github.com/redis/hiredis/pull/644) ([codehz](https://github.com/codehz)) +- Makefile: use predefined AR [\#632](https://github.com/redis/hiredis/pull/632) ([Mic92](https://github.com/Mic92)) +- FreeBSD build fix [\#628](https://github.com/redis/hiredis/pull/628) ([devnexen](https://github.com/devnexen)) +- Fix errors not propagating properly with libuv.h. [\#624](https://github.com/redis/hiredis/pull/624) ([yossigo](https://github.com/yossigo)) +- Update README.md [\#621](https://github.com/redis/hiredis/pull/621) ([Crunsher](https://github.com/Crunsher)) +- Fix redisBufferRead documentation [\#620](https://github.com/redis/hiredis/pull/620) ([hacst](https://github.com/hacst)) +- Add CPPFLAGS to REAL\_CFLAGS [\#614](https://github.com/redis/hiredis/pull/614) ([thomaslee](https://github.com/thomaslee)) +- Update createArray to take size\_t [\#597](https://github.com/redis/hiredis/pull/597) ([justinbrewer](https://github.com/justinbrewer)) +- fix common realloc mistake and add null check more [\#580](https://github.com/redis/hiredis/pull/580) ([charsyam](https://github.com/charsyam)) +- Proper error reporting for connect failures [\#578](https://github.com/redis/hiredis/pull/578) ([mnunberg](https://github.com/mnunberg)) + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* + +## [1.0.0-rc1](https://github.com/redis/hiredis/tree/v1.0.0-rc1) - (2020-07-29) + +_Note: There were no changes to code between v1.0.0-rc1 and v1.0.0 so see v1.0.0 for changelog_ + +### 0.14.1 (2020-03-13) + +* Adds safe allocation wrappers (CVE-2020-7105, #747, #752) (Michael Grunder) + ### 0.14.0 (2018-09-25) +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + + User code should compare this to `size_t` values as well. + If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. * Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) * Use -dynamiclib instead of -shared for OSX (Ryan Schmidt [a65537]) @@ -196,4 +343,3 @@ The parser, standalone since v0.12.0, can now be compiled on Windows ### 0.10.0 * See commit log. - diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt index 9e78894f3..f86c9b70b 100644 --- a/deps/hiredis/CMakeLists.txt +++ b/deps/hiredis/CMakeLists.txt @@ -3,6 +3,8 @@ INCLUDE(GNUInstallDirs) PROJECT(hiredis) OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) +OPTION(DISABLE_TESTS "If tests should be compiled or not" OFF) +OPTION(ENABLE_SSL_TESTS, "Should we test SSL connections" OFF) MACRO(getVersionBit name) SET(VERSION_REGEX "^#define ${name} (.+)$") @@ -22,7 +24,8 @@ PROJECT(hiredis VERSION "${VERSION}") SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") -ADD_LIBRARY(hiredis SHARED +SET(hiredis_sources + alloc.c async.c dict.c hiredis.c @@ -31,20 +34,32 @@ ADD_LIBRARY(hiredis SHARED sds.c sockcompat.c) +SET(hiredis_sources ${hiredis_sources}) + +IF(WIN32) + ADD_COMPILE_DEFINITIONS(_CRT_SECURE_NO_WARNINGS WIN32_LEAN_AND_MEAN) +ENDIF() + +ADD_LIBRARY(hiredis SHARED ${hiredis_sources}) + SET_TARGET_PROPERTIES(hiredis - PROPERTIES + PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE VERSION "${HIREDIS_SONAME}") IF(WIN32 OR MINGW) TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) ENDIF() -TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $ $) CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) INSTALL(TARGETS hiredis - DESTINATION "${CMAKE_INSTALL_LIBDIR}") + EXPORT hiredis-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -INSTALL(FILES hiredis.h read.h sds.h async.h +INSTALL(FILES hiredis.h read.h sds.h async.h alloc.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) INSTALL(DIRECTORY adapters @@ -53,6 +68,26 @@ INSTALL(DIRECTORY adapters INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +export(EXPORT hiredis-targets + FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis-targets.cmake" + NAMESPACE hiredis::) + +SET(CMAKE_CONF_INSTALL_DIR share/hiredis) +SET(INCLUDE_INSTALL_DIR include) +include(CMakePackageConfigHelpers) +configure_package_config_file(hiredis-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake + INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR} + PATH_VARS INCLUDE_INSTALL_DIR) + +INSTALL(EXPORT hiredis-targets + FILE hiredis-targets.cmake + NAMESPACE hiredis:: + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + + IF(ENABLE_SSL) IF (NOT OPENSSL_ROOT_DIR) IF (APPLE) @@ -60,26 +95,66 @@ IF(ENABLE_SSL) ENDIF() ENDIF() FIND_PACKAGE(OpenSSL REQUIRED) - ADD_LIBRARY(hiredis_ssl SHARED + SET(hiredis_ssl_sources ssl.c) + ADD_LIBRARY(hiredis_ssl SHARED + ${hiredis_ssl_sources}) + + IF (APPLE) + SET_PROPERTY(TARGET hiredis_ssl PROPERTY LINK_FLAGS "-Wl,-undefined -Wl,dynamic_lookup") + ENDIF() + + SET_TARGET_PROPERTIES(hiredis_ssl + PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS TRUE + VERSION "${HIREDIS_SONAME}") + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + IF (WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE hiredis) + ENDIF() CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) INSTALL(TARGETS hiredis_ssl - DESTINATION "${CMAKE_INSTALL_LIBDIR}") + EXPORT hiredis_ssl-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES hiredis_ssl.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + + export(EXPORT hiredis_ssl-targets + FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-targets.cmake" + NAMESPACE hiredis::) + + SET(CMAKE_CONF_INSTALL_DIR share/hiredis_ssl) + configure_package_config_file(hiredis_ssl-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake + INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR} + PATH_VARS INCLUDE_INSTALL_DIR) + + INSTALL(EXPORT hiredis_ssl-targets + FILE hiredis_ssl-targets.cmake + NAMESPACE hiredis:: + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) ENDIF() -IF(NOT (WIN32 OR MINGW)) +IF(NOT DISABLE_TESTS) ENABLE_TESTING() ADD_EXECUTABLE(hiredis-test test.c) - TARGET_LINK_LIBRARIES(hiredis-test hiredis) + IF(ENABLE_SSL_TESTS) + ADD_DEFINITIONS(-DHIREDIS_TEST_SSL=1) + TARGET_LINK_LIBRARIES(hiredis-test hiredis hiredis_ssl) + ELSE() + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ENDIF() ADD_TEST(NAME hiredis-test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) ENDIF() diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 841990d5c..a9e87dd82 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,16 +3,16 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +OBJ=alloc.o net.o hiredis.o sds.o async.o read.o sockcompat.o SSL_OBJ=ssl.o -EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib hiredis-example-push ifeq ($(USE_SSL),1) EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl endif TESTS=hiredis-test LIBNAME=libhiredis -SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_LIBNAME=libhiredis_ssl SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') @@ -55,12 +55,17 @@ STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) + DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) STLIB_MAKE_CMD=$(AR) rcs +SSL_DYLIB_MINOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) +SSL_DYLIB_MAJOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +SSL_DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(SSL_DYLIB_MINOR_NAME) + # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') @@ -80,13 +85,22 @@ else endif ifeq ($(uname_S),SunOS) + IS_SUN_CC=$(shell sh -c '$(CC) -V 2>&1 |egrep -i -c "sun|studio"') + ifeq ($(IS_SUN_CC),1) + SUN_SHARED_FLAG=-G + else + SUN_SHARED_FLAG=-shared + endif REAL_LDFLAGS+= -ldl -lnsl -lsocket - DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) + DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) + SSL_DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(SSL_DYLIBNAME) -h $(SSL_DYLIB_MINOR_NAME) $(LDFLAGS) $(SSL_LDFLAGS) endif ifeq ($(uname_S),Darwin) DYLIBSUFFIX=dylib DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX) DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) + SSL_DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) -o $(SSL_DYLIBNAME) $(LDFLAGS) $(SSL_LDFLAGS) + DYLIB_PLUGIN=-Wl,-undefined -Wl,dynamic_lookup endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) @@ -95,15 +109,16 @@ all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) endif # Deps (use make dep to generate this) -async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h -dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h -read.o: read.c fmacros.h read.h sds.h -sds.o: sds.c sds.h +alloc.o: alloc.c fmacros.h alloc.h +async.o: async.c fmacros.h alloc.h async.h hiredis.h read.h sds.h net.h dict.c dict.h win32.h async_private.h +dict.o: dict.c fmacros.h alloc.h dict.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h alloc.h net.h async.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h alloc.h sockcompat.h win32.h +read.o: read.c fmacros.h alloc.h read.h sds.h win32.h +sds.o: sds.c sds.h sdsalloc.h alloc.h sockcompat.o: sockcompat.c sockcompat.h -ssl.o: ssl.c hiredis.h -test.o: test.c fmacros.h hiredis.h read.h sds.h +ssl.o: ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h +test.o: test.c fmacros.h hiredis.h read.h sds.h alloc.h net.h sockcompat.h win32.h $(DYLIBNAME): $(OBJ) $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) @@ -112,7 +127,7 @@ $(STLIBNAME): $(OBJ) $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) $(SSL_DYLIBNAME): $(SSL_OBJ) - $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + $(SSL_DYLIB_MAKE_CMD) $(DYLIB_PLUGIN) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(LDFLAGS) $(SSL_LDFLAGS) $(SSL_STLIBNAME): $(SSL_OBJ) $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) @@ -146,6 +161,7 @@ hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + ifndef AE_DIR hiredis-example-ae: @echo "Please specify AE_DIR (e.g. /src)" @@ -180,13 +196,19 @@ endif hiredis-example: examples/example.c $(STLIBNAME) $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) +hiredis-example-push: examples/example-push.c $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) + examples: $(EXAMPLES) TEST_LIBS = $(STLIBNAME) ifeq ($(USE_SSL),1) - TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread + TEST_LIBS += $(SSL_STLIBNAME) + TEST_LDFLAGS = $(SSL_LDFLAGS) -lssl -lcrypto -lpthread endif + hiredis-test: test.o $(TEST_LIBS) + $(CC) -o $@ $(REAL_CFLAGS) -I. $^ $(REAL_LDFLAGS) $(TEST_LDFLAGS) hiredis-%: %.o $(STLIBNAME) $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) @@ -221,7 +243,7 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ -$(SSL_PKGCONFNAME): hiredis.h +$(SSL_PKGCONFNAME): hiredis_ssl.h @echo "Generating $@ for pkgconfig..." @echo prefix=$(PREFIX) > $@ @echo exec_prefix=\$${prefix} >> $@ @@ -237,7 +259,7 @@ $(SSL_PKGCONFNAME): hiredis.h install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) - $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) + $(INSTALL) hiredis.h async.h read.h sds.h alloc.h $(INSTALL_INCLUDE_PATH) $(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME) cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) @@ -245,6 +267,19 @@ install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_PKGCONF_PATH) $(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH) +ifeq ($(USE_SSL),1) +install: install-ssl + +install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) + mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) + $(INSTALL) hiredis_ssl.h $(INSTALL_INCLUDE_PATH) + $(INSTALL) $(SSL_DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) + cd $(INSTALL_LIBRARY_PATH) && ln -sf $(SSL_DYLIB_MINOR_NAME) $(SSL_DYLIBNAME) + $(INSTALL) $(SSL_STLIBNAME) $(INSTALL_LIBRARY_PATH) + mkdir -p $(INSTALL_PKGCONF_PATH) + $(INSTALL) $(SSL_PKGCONFNAME) $(INSTALL_PKGCONF_PATH) +endif + 32bit: @echo "" @echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386" diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index c0b432f07..3a22553ea 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -1,6 +1,6 @@ [![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis) -**This Readme reflects the latest changed in the master branch. See [v0.13.3](https://github.com/redis/hiredis/tree/v0.13.3) for the Readme and documentation for the latest release.** +**This Readme reflects the latest changed in the master branch. See [v1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) for the Readme and documentation for the latest release ([API/ABI history](https://abi-laboratory.pro/?view=timeline&l=hiredis)).** # HIREDIS @@ -24,12 +24,31 @@ The library comes with multiple APIs. There is the ## Upgrading to `1.0.0` -Version 1.0.0 marks a stable release of hiredis. +Version 1.0.0 marks the first stable release of Hiredis. It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory. It also bundles the updated `sds` library, to sync up with upstream and Redis. -For most applications a recompile against the new hiredis should be enough. For code changes see the [Changelog](CHANGELOG.md). +_Note: As described below, a few member names have been changed but most applications should be able to upgrade with minor code changes and recompiling._ + +## IMPORTANT: Breaking changes from `0.14.1` -> `1.0.0` + +* `redisContext` has two additional members (`free_privdata`, and `privctx`). +* `redisOptions.timeout` has been renamed to `redisOptions.connect_timeout`, and we've added `redisOptions.command_timeout`. +* `redisReplyObjectFunctions.createArray` now takes `size_t` instead of `int` for its length parameter. + +## IMPORTANT: Breaking changes when upgrading from 0.13.x -> 0.14.x + +Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now +protocol errors. This is consistent with the RESP specification. On 32-bit +platforms, the upper bound is lowered to `SIZE_MAX`. + +Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. If it was used to +compare to other values, casting might be necessary or can be removed, if +casting was applied before. + ## Upgrading from `<0.9.0` Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing @@ -110,6 +129,8 @@ The standard replies that `redisCommand` are of the type `redisReply`. The `type` field in the `redisReply` should be used to test what kind of reply was received: +### RESP2 + * **`REDIS_REPLY_STATUS`**: * The command replied with a status reply. The status string can be accessed using `reply->str`. The length of this string can be accessed using `reply->len`. @@ -134,16 +155,51 @@ was received: and can be accessed via `reply->element[..index..]`. Redis may reply with nested arrays but this is fully supported. +### RESP3 + +Hiredis also supports every new `RESP3` data type which are as follows. For more information about the protocol see the `RESP3` [specification.](https://github.com/antirez/RESP3/blob/master/spec.md) + +* **`REDIS_REPLY_DOUBLE`**: + * The command replied with a double-precision floating point number. + The value is stored as a string in the `str` member, and can be converted with `strtod` or similar. + +* **`REDIS_REPLY_BOOL`**: + * A boolean true/false reply. + The value is stored in the `integer` member and will be either `0` or `1`. + +* **`REDIS_REPLY_MAP`**: + * An array with the added invariant that there will always be an even number of elements. + The MAP is functionally equivelant to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant. + +* **`REDIS_REPLY_SET`**: + * An array response where each entry is unique. + Like the MAP type, the data is identical to an array response except there are no duplicate values. + +* **`REDIS_REPLY_PUSH`**: + * An array that can be generated spontaneously by Redis. + This array response will always contain at least two subelements. The first contains the type of `PUSH` message (e.g. `message`, or `invalidate`), and the second being a sub-array with the `PUSH` payload itself. + +* **`REDIS_REPLY_ATTR`**: + * An array structurally identical to a `MAP` but intended as meta-data about a reply. + _As of Redis 6.0.6 this reply type is not used in Redis_ + +* **`REDIS_REPLY_BIGNUM`**: + * A string representing an arbitrarily large signed or unsigned integer value. + The number will be encoded as a string in the `str` member of `redisReply`. + +* **`REDIS_REPLY_VERB`**: + * A verbatim string, intended to be presented to the user without modification. + The string payload is stored in the `str` memeber, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown). + Replies should be freed using the `freeReplyObject()` function. Note that this function will take care of freeing sub-reply objects contained in arrays and nested arrays, so there is no need for the user to free the sub replies (it is actually harmful and will corrupt the memory). -**Important:** the current version of hiredis (0.10.0) frees replies when the +**Important:** the current version of hiredis (1.0.0) frees replies when the asynchronous API is used. This means you should not call `freeReplyObject` when you use this API. The reply is cleaned up by hiredis _after_ the callback -returns. This behavior will probably change in future releases, so make sure to -keep an eye on the changelog when upgrading (see issue #39). +returns. We may introduce a flag to make this configurable in future versions of the library. ### Cleaning up @@ -205,16 +261,16 @@ a single call to `read(2)`): redisReply *reply; redisAppendCommand(context,"SET foo bar"); redisAppendCommand(context,"GET foo"); -redisGetReply(context,&reply); // reply for SET +redisGetReply(context,(void *)&reply); // reply for SET freeReplyObject(reply); -redisGetReply(context,&reply); // reply for GET +redisGetReply(context,(void *)&reply); // reply for GET freeReplyObject(reply); ``` This API can also be used to implement a blocking subscriber: ```c reply = redisCommand(context,"SUBSCRIBE foo"); freeReplyObject(reply); -while(redisGetReply(context,&reply) == REDIS_OK) { +while(redisGetReply(context,(void *)&reply) == REDIS_OK) { // consume message freeReplyObject(reply); } @@ -404,9 +460,199 @@ This should be done only in order to maximize performances when working with large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again as soon as possible in order to prevent allocation of useless memory. +### Reader max array elements + +By default the hiredis reply parser sets the maximum number of multi-bulk elements +to 2^32 - 1 or 4,294,967,295 entries. If you need to process multi-bulk replies +with more than this many elements you can set the value higher or to zero, meaning +unlimited with: +```c +context->reader->maxelements = 0; +``` + +## SSL/TLS Support + +### Building + +SSL/TLS support is not built by default and requires an explicit flag: + + make USE_SSL=1 + +This requires OpenSSL development package (e.g. including header files to be +available. + +When enabled, SSL/TLS support is built into extra `libhiredis_ssl.a` and +`libhiredis_ssl.so` static/dynamic libraries. This leaves the original libraries +unaffected so no additional dependencies are introduced. + +### Using it + +First, you'll need to make sure you include the SSL header file: + +```c +#include "hiredis.h" +#include "hiredis_ssl.h" +``` + +You will also need to link against `libhiredis_ssl`, **in addition** to +`libhiredis` and add `-lssl -lcrypto` to satisfy its dependencies. + +Hiredis implements SSL/TLS on top of its normal `redisContext` or +`redisAsyncContext`, so you will need to establish a connection first and then +initiate an SSL/TLS handshake. + +#### Hiredis OpenSSL Wrappers + +Before Hiredis can negotiate an SSL/TLS connection, it is necessary to +initialize OpenSSL and create a context. You can do that in two ways: + +1. Work directly with the OpenSSL API to initialize the library's global context + and create `SSL_CTX *` and `SSL *` contexts. With an `SSL *` object you can + call `redisInitiateSSL()`. +2. Work with a set of Hiredis-provided wrappers around OpenSSL, create a + `redisSSLContext` object to hold configuration and use + `redisInitiateSSLWithContext()` to initiate the SSL/TLS handshake. + +```c +/* An Hiredis SSL context. It holds SSL configuration and can be reused across + * many contexts. + */ +redisSSLContext *ssl; + +/* An error variable to indicate what went wrong, if the context fails to + * initialize. + */ +redisSSLContextError ssl_error; + +/* Initialize global OpenSSL state. + * + * You should call this only once when your app initializes, and only if + * you don't explicitly or implicitly initialize OpenSSL it elsewhere. + */ +redisInitOpenSSL(); + +/* Create SSL context */ +ssl = redisCreateSSLContext( + "cacertbundle.crt", /* File name of trusted CA/ca bundle file, optional */ + "/path/to/certs", /* Path of trusted certificates, optional */ + "client_cert.pem", /* File name of client certificate file, optional */ + "client_key.pem", /* File name of client private key, optional */ + "redis.mydomain.com", /* Server name to request (SNI), optional */ + &ssl_error + ) != REDIS_OK) { + printf("SSL error: %s\n", redisSSLContextGetError(ssl_error); + /* Abort... */ + } + +/* Create Redis context and establish connection */ +c = redisConnect("localhost", 6443); +if (c == NULL || c->err) { + /* Handle error and abort... */ +} + +/* Negotiate SSL/TLS */ +if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) { + /* Handle error, in c->err / c->errstr */ +} +``` + +## RESP3 PUSH replies +Redis 6.0 introduced PUSH replies with the reply-type `>`. These messages are generated spontaneously and can arrive at any time, so must be handled using callbacks. + +### Default behavior +Hiredis installs handlers on `redisContext` and `redisAsyncContext` by default, which will intercept and free any PUSH replies detected. This means existing code will work as-is after upgrading to Redis 6 and switching to `RESP3`. + +### Custom PUSH handler prototypes +The callback prototypes differ between `redisContext` and `redisAsyncContext`. + +#### redisContext +```c +void my_push_handler(void *privdata, void *reply) { + /* Handle the reply */ + + /* Note: We need to free the reply in our custom handler for + blocking contexts. This lets us keep the reply if + we want. */ + freeReplyObject(reply); +} +``` + +#### redisAsyncContext +```c +void my_async_push_handler(redisAsyncContext *ac, void *reply) { + /* Handle the reply */ + + /* Note: Because async hiredis always frees replies, you should + not call freeReplyObject in an async push callback. */ +} +``` + +### Installing a custom handler +There are two ways to set your own PUSH handlers. + +1. Set `push_cb` or `async_push_cb` in the `redisOptions` struct and connect with `redisConnectWithOptions` or `redisAsyncConnectWithOptions`. + ```c + redisOptions = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + options->push_cb = my_push_handler; + redisContext *context = redisConnectWithOptions(&options); + ``` +2. Call `redisSetPushCallback` or `redisAsyncSetPushCallback` on a connected context. + ```c + redisContext *context = redisConnect("127.0.0.1", 6379); + redisSetPushCallback(context, my_push_handler); + ``` + + _Note `redisSetPushCallback` and `redisAsyncSetPushCallback` both return any currently configured handler, making it easy to override and then return to the old value._ + +### Specifying no handler +If you have a unique use-case where you don't want hiredis to automatically intercept and free PUSH replies, you will want to configure no handler at all. This can be done in two ways. +1. Set the `REDIS_OPT_NO_PUSH_AUTOFREE` flag in `redisOptions` and leave the callback function pointer `NULL`. + ```c + redisOptions = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + options->options |= REDIS_OPT_NO_PUSH_AUTOFREE; + redisContext *context = redisConnectWithOptions(&options); + ``` +3. Call `redisSetPushCallback` with `NULL` once connected. + ```c + redisContext *context = redisConnect("127.0.0.1", 6379); + redisSetPushCallback(context, NULL); + ``` + + _Note: With no handler configured, calls to `redisCommand` may generate more than one reply, so this strategy is only applicable when there's some kind of blocking`redisGetReply()` loop (e.g. `MONITOR` or `SUBSCRIBE` workloads)._ + +## Allocator injection + +Hiredis uses a pass-thru structure of function pointers defined in [alloc.h](https://github.com/redis/hiredis/blob/f5d25850/alloc.h#L41) that contain the currently configured allocation and deallocation functions. By default they just point to libc (`malloc`, `calloc`, `realloc`, etc). + +### Overriding + +One can override the allocators like so: + +```c +hiredisAllocFuncs myfuncs = { + .mallocFn = my_malloc, + .callocFn = my_calloc, + .reallocFn = my_realloc, + .strdupFn = my_strdup, + .freeFn = my_free, +}; + +// Override allocators (function returns current allocators if needed) +hiredisAllocFuncs orig = hiredisSetAllocators(&myfuncs); +``` + +To reset the allocators to their default libc function simply call: + +```c +hiredisResetAllocators(); +``` + ## AUTHORS -Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. -Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and -Jan-Erik Rediger (janerik at fnordig dot com) +Salvatore Sanfilippo (antirez at gmail),\ +Pieter Noordhuis (pcnoordhuis at gmail)\ +Michael Grunder (michael dot grunder at gmail) + +_Hiredis is released under the BSD license._ diff --git a/deps/hiredis/adapters/ae.h b/deps/hiredis/adapters/ae.h index 5c551c2ed..660d82eb0 100644 --- a/deps/hiredis/adapters/ae.h +++ b/deps/hiredis/adapters/ae.h @@ -96,7 +96,7 @@ static void redisAeCleanup(void *privdata) { redisAeEvents *e = (redisAeEvents*)privdata; redisAeDelRead(privdata); redisAeDelWrite(privdata); - free(e); + hi_free(e); } static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { @@ -108,7 +108,10 @@ static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisAeEvents*)malloc(sizeof(*e)); + e = (redisAeEvents*)hi_malloc(sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; e->loop = loop; e->fd = c->fd; diff --git a/deps/hiredis/adapters/glib.h b/deps/hiredis/adapters/glib.h index e0a6411d3..ad59dd142 100644 --- a/deps/hiredis/adapters/glib.h +++ b/deps/hiredis/adapters/glib.h @@ -134,6 +134,9 @@ redis_source_new (redisAsyncContext *ac) g_return_val_if_fail(ac != NULL, NULL); source = (RedisSource *)g_source_new(&source_funcs, sizeof *source); + if (source == NULL) + return NULL; + source->ac = ac; source->poll_fd.fd = c->fd; source->poll_fd.events = 0; diff --git a/deps/hiredis/adapters/ivykis.h b/deps/hiredis/adapters/ivykis.h index 6a12a868a..179f6ab52 100644 --- a/deps/hiredis/adapters/ivykis.h +++ b/deps/hiredis/adapters/ivykis.h @@ -43,7 +43,7 @@ static void redisIvykisCleanup(void *privdata) { redisIvykisEvents *e = (redisIvykisEvents*)privdata; iv_fd_unregister(&e->fd); - free(e); + hi_free(e); } static int redisIvykisAttach(redisAsyncContext *ac) { @@ -55,7 +55,10 @@ static int redisIvykisAttach(redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisIvykisEvents*)malloc(sizeof(*e)); + e = (redisIvykisEvents*)hi_malloc(sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; /* Register functions to start/stop listening for events */ diff --git a/deps/hiredis/adapters/libev.h b/deps/hiredis/adapters/libev.h index 2bf8d521f..e1e7bbd99 100644 --- a/deps/hiredis/adapters/libev.h +++ b/deps/hiredis/adapters/libev.h @@ -41,6 +41,7 @@ typedef struct redisLibevEvents { struct ev_loop *loop; int reading, writing; ev_io rev, wev; + ev_timer timer; } redisLibevEvents; static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) { @@ -103,11 +104,39 @@ static void redisLibevDelWrite(void *privdata) { } } +static void redisLibevStopTimer(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + ev_timer_stop(EV_A_ &e->timer); +} + static void redisLibevCleanup(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; redisLibevDelRead(privdata); redisLibevDelWrite(privdata); - free(e); + redisLibevStopTimer(privdata); + hi_free(e); +} + +static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) { + ((void)revents); + redisLibevEvents *e = (redisLibevEvents*)timer->data; + redisAsyncHandleTimeout(e->context); +} + +static void redisLibevSetTimeout(void *privdata, struct timeval tv) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + + if (!ev_is_active(&e->timer)) { + ev_init(&e->timer, redisLibevTimeout); + e->timer.data = e; + } + + e->timer.repeat = tv.tv_sec + tv.tv_usec / 1000000.00; + ev_timer_again(EV_A_ &e->timer); } static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { @@ -119,14 +148,16 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibevEvents*)malloc(sizeof(*e)); + e = (redisLibevEvents*)hi_calloc(1, sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; #if EV_MULTIPLICITY e->loop = loop; #else e->loop = NULL; #endif - e->reading = e->writing = 0; e->rev.data = e; e->wev.data = e; @@ -136,6 +167,7 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { ac->ev.addWrite = redisLibevAddWrite; ac->ev.delWrite = redisLibevDelWrite; ac->ev.cleanup = redisLibevCleanup; + ac->ev.scheduleTimer = redisLibevSetTimeout; ac->ev.data = e; /* Initialize read/write events */ diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index a4952776c..9150979bc 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -47,7 +47,7 @@ typedef struct redisLibeventEvents { } redisLibeventEvents; static void redisLibeventDestroy(redisLibeventEvents *e) { - free(e); + hi_free(e); } static void redisLibeventHandler(int fd, short event, void *arg) { @@ -152,7 +152,10 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)calloc(1, sizeof(*e)); + e = (redisLibeventEvents*)hi_calloc(1, sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; /* Register functions to start/stop listening for events */ diff --git a/deps/hiredis/adapters/libuv.h b/deps/hiredis/adapters/libuv.h index 39ef7cf5e..c120b1b39 100644 --- a/deps/hiredis/adapters/libuv.h +++ b/deps/hiredis/adapters/libuv.h @@ -73,7 +73,7 @@ static void redisLibuvDelWrite(void *privdata) { static void on_close(uv_handle_t* handle) { redisLibuvEvents* p = (redisLibuvEvents*)handle->data; - free(p); + hi_free(p); } @@ -98,15 +98,13 @@ static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { ac->ev.delWrite = redisLibuvDelWrite; ac->ev.cleanup = redisLibuvCleanup; - redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p)); - - if (!p) { - return REDIS_ERR; - } + redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p)); + if (p == NULL) + return REDIS_ERR; memset(p, 0, sizeof(*p)); - if (uv_poll_init(loop, &p->handle, c->fd) != 0) { + if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) { return REDIS_ERR; } diff --git a/deps/hiredis/adapters/macosx.h b/deps/hiredis/adapters/macosx.h index 72121f606..3c87f1b2f 100644 --- a/deps/hiredis/adapters/macosx.h +++ b/deps/hiredis/adapters/macosx.h @@ -27,7 +27,7 @@ static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) { CFSocketInvalidate(redisRunLoop->socketRef); CFRelease(redisRunLoop->socketRef); } - free(redisRunLoop); + hi_free(redisRunLoop); } return REDIS_ERR; } @@ -80,8 +80,9 @@ static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLo /* Nothing should be attached when something is already attached */ if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR; - RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop)); - if( !redisRunLoop ) return REDIS_ERR; + RedisRunLoop* redisRunLoop = (RedisRunLoop*) hi_calloc(1, sizeof(RedisRunLoop)); + if (redisRunLoop == NULL) + return REDIS_ERR; /* Setup redis stuff */ redisRunLoop->context = redisAsyncCtx; diff --git a/deps/hiredis/alloc.c b/deps/hiredis/alloc.c new file mode 100644 index 000000000..7fb6b35e7 --- /dev/null +++ b/deps/hiredis/alloc.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "fmacros.h" +#include "alloc.h" +#include +#include + +hiredisAllocFuncs hiredisAllocFns = { + .mallocFn = malloc, + .callocFn = calloc, + .reallocFn = realloc, + .strdupFn = strdup, + .freeFn = free, +}; + +/* Override hiredis' allocators with ones supplied by the user */ +hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *override) { + hiredisAllocFuncs orig = hiredisAllocFns; + + hiredisAllocFns = *override; + + return orig; +} + +/* Reset allocators to use libc defaults */ +void hiredisResetAllocators(void) { + hiredisAllocFns = (hiredisAllocFuncs) { + .mallocFn = malloc, + .callocFn = calloc, + .reallocFn = realloc, + .strdupFn = strdup, + .freeFn = free, + }; +} + +#ifdef _WIN32 + +void *hi_malloc(size_t size) { + return hiredisAllocFns.mallocFn(size); +} + +void *hi_calloc(size_t nmemb, size_t size) { + return hiredisAllocFns.callocFn(nmemb, size); +} + +void *hi_realloc(void *ptr, size_t size) { + return hiredisAllocFns.reallocFn(ptr, size); +} + +char *hi_strdup(const char *str) { + return hiredisAllocFns.strdupFn(str); +} + +void hi_free(void *ptr) { + hiredisAllocFns.freeFn(ptr); +} + +#endif diff --git a/deps/hiredis/alloc.h b/deps/hiredis/alloc.h new file mode 100644 index 000000000..34a05f49f --- /dev/null +++ b/deps/hiredis/alloc.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef HIREDIS_ALLOC_H +#define HIREDIS_ALLOC_H + +#include /* for size_t */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure pointing to our actually configured allocators */ +typedef struct hiredisAllocFuncs { + void *(*mallocFn)(size_t); + void *(*callocFn)(size_t,size_t); + void *(*reallocFn)(void*,size_t); + char *(*strdupFn)(const char*); + void (*freeFn)(void*); +} hiredisAllocFuncs; + +hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *ha); +void hiredisResetAllocators(void); + +#ifndef _WIN32 + +/* Hiredis' configured allocator function pointer struct */ +extern hiredisAllocFuncs hiredisAllocFns; + +static inline void *hi_malloc(size_t size) { + return hiredisAllocFns.mallocFn(size); +} + +static inline void *hi_calloc(size_t nmemb, size_t size) { + return hiredisAllocFns.callocFn(nmemb, size); +} + +static inline void *hi_realloc(void *ptr, size_t size) { + return hiredisAllocFns.reallocFn(ptr, size); +} + +static inline char *hi_strdup(const char *str) { + return hiredisAllocFns.strdupFn(str); +} + +static inline void hi_free(void *ptr) { + hiredisAllocFns.freeFn(ptr); +} + +#else + +void *hi_malloc(size_t size); +void *hi_calloc(size_t nmemb, size_t size); +void *hi_realloc(void *ptr, size_t size); +char *hi_strdup(const char *str); +void hi_free(void *ptr); + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* HIREDIS_ALLOC_H */ diff --git a/deps/hiredis/async.c b/deps/hiredis/async.c index 4f422d566..64ab601c9 100644 --- a/deps/hiredis/async.c +++ b/deps/hiredis/async.c @@ -30,6 +30,7 @@ */ #include "fmacros.h" +#include "alloc.h" #include #include #ifndef _MSC_VER @@ -46,18 +47,24 @@ #include "async_private.h" -/* Forward declaration of function in hiredis.c */ +/* Forward declarations of hiredis.c functions */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); +void __redisSetError(redisContext *c, int type, const char *str); /* Functions managing dictionary of callbacks for pub/sub. */ static unsigned int callbackHash(const void *key) { return dictGenHashFunction((const unsigned char *)key, - sdslen((const sds)key)); + hi_sdslen((const hisds)key)); } static void *callbackValDup(void *privdata, const void *src) { ((void) privdata); - redisCallback *dup = malloc(sizeof(*dup)); + redisCallback *dup; + + dup = hi_malloc(sizeof(*dup)); + if (dup == NULL) + return NULL; + memcpy(dup,src,sizeof(*dup)); return dup; } @@ -66,20 +73,20 @@ static int callbackKeyCompare(void *privdata, const void *key1, const void *key2 int l1, l2; ((void) privdata); - l1 = sdslen((const sds)key1); - l2 = sdslen((const sds)key2); + l1 = hi_sdslen((const hisds)key1); + l2 = hi_sdslen((const hisds)key2); if (l1 != l2) return 0; return memcmp(key1,key2,l1) == 0; } static void callbackKeyDestructor(void *privdata, void *key) { ((void) privdata); - sdsfree((sds)key); + hi_sdsfree((hisds)key); } static void callbackValDestructor(void *privdata, void *val) { ((void) privdata); - free(val); + hi_free(val); } static dictType callbackDict = { @@ -93,10 +100,19 @@ static dictType callbackDict = { static redisAsyncContext *redisAsyncInitialize(redisContext *c) { redisAsyncContext *ac; + dict *channels = NULL, *patterns = NULL; - ac = realloc(c,sizeof(redisAsyncContext)); + channels = dictCreate(&callbackDict,NULL); + if (channels == NULL) + goto oom; + + patterns = dictCreate(&callbackDict,NULL); + if (patterns == NULL) + goto oom; + + ac = hi_realloc(c,sizeof(redisAsyncContext)); if (ac == NULL) - return NULL; + goto oom; c = &(ac->c); @@ -108,6 +124,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->err = 0; ac->errstr = NULL; ac->data = NULL; + ac->dataCleanup = NULL; ac->ev.data = NULL; ac->ev.addRead = NULL; @@ -124,9 +141,14 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->replies.tail = NULL; ac->sub.invalid.head = NULL; ac->sub.invalid.tail = NULL; - ac->sub.channels = dictCreate(&callbackDict,NULL); - ac->sub.patterns = dictCreate(&callbackDict,NULL); + ac->sub.channels = channels; + ac->sub.patterns = patterns; + return ac; +oom: + if (channels) dictRelease(channels); + if (patterns) dictRelease(patterns); + return NULL; } /* We want the error field to be accessible directly instead of requiring @@ -145,16 +167,26 @@ redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { redisContext *c; redisAsyncContext *ac; + /* Clear any erroneously set sync callback and flag that we don't want to + * use freeReplyObject by default. */ + myOptions.push_cb = NULL; + myOptions.options |= REDIS_OPT_NO_PUSH_AUTOFREE; + myOptions.options |= REDIS_OPT_NONBLOCK; c = redisConnectWithOptions(&myOptions); if (c == NULL) { return NULL; } + ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } + + /* Set any configured async push handler */ + redisAsyncSetPushCallback(ac, myOptions.async_push_cb); + __redisAsyncCopyError(ac); return ac; } @@ -214,7 +246,7 @@ static int __redisPushCallback(redisCallbackList *list, redisCallback *source) { redisCallback *cb; /* Copy callback from stack to heap */ - cb = malloc(sizeof(*cb)); + cb = hi_malloc(sizeof(*cb)); if (cb == NULL) return REDIS_ERR_OOM; @@ -242,7 +274,7 @@ static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) /* Copy callback from heap to stack */ if (target != NULL) memcpy(target,cb,sizeof(*cb)); - free(cb); + hi_free(cb); return REDIS_OK; } return REDIS_ERR; @@ -257,6 +289,14 @@ static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisRe } } +static void __redisRunPushCallback(redisAsyncContext *ac, redisReply *reply) { + if (ac->push_cb != NULL) { + ac->c.flags |= REDIS_IN_CALLBACK; + ac->push_cb(ac, reply); + ac->c.flags &= ~REDIS_IN_CALLBACK; + } +} + /* Helper function to free the context. */ static void __redisAsyncFree(redisAsyncContext *ac) { redisContext *c = &(ac->c); @@ -272,18 +312,28 @@ static void __redisAsyncFree(redisAsyncContext *ac) { while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK) __redisRunCallback(ac,&cb,NULL); - /* Run subscription callbacks callbacks with NULL reply */ - it = dictGetIterator(ac->sub.channels); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.channels); + /* Run subscription callbacks with NULL reply */ + if (ac->sub.channels) { + it = dictGetIterator(ac->sub.channels); + if (it != NULL) { + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + } - it = dictGetIterator(ac->sub.patterns); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.patterns); + dictRelease(ac->sub.channels); + } + + if (ac->sub.patterns) { + it = dictGetIterator(ac->sub.patterns); + if (it != NULL) { + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + } + + dictRelease(ac->sub.patterns); + } /* Signal event lib to clean up */ _EL_CLEANUP(ac); @@ -298,6 +348,10 @@ static void __redisAsyncFree(redisAsyncContext *ac) { } } + if (ac->dataCleanup) { + ac->dataCleanup(ac->data); + } + /* Cleanup self */ redisFree(c); } @@ -364,11 +418,11 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, dictEntry *de; int pvariant; char *stype; - sds sname; + hisds sname; /* Custom reply functions are not supported for pub/sub. This will fail * very hard when they are used... */ - if (reply->type == REDIS_REPLY_ARRAY) { + if (reply->type == REDIS_REPLY_ARRAY || reply->type == REDIS_REPLY_PUSH) { assert(reply->elements >= 2); assert(reply->element[0]->type == REDIS_REPLY_STRING); stype = reply->element[0]->str; @@ -381,7 +435,10 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, /* Locate the right callback */ assert(reply->element[1]->type == REDIS_REPLY_STRING); - sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len); + sname = hi_sdsnewlen(reply->element[1]->str,reply->element[1]->len); + if (sname == NULL) + goto oom; + de = dictFind(callbacks,sname); if (de != NULL) { cb = dictGetEntryVal(de); @@ -409,12 +466,39 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, c->flags &= ~REDIS_SUBSCRIBED; } } - sdsfree(sname); + hi_sdsfree(sname); } else { /* Shift callback for invalid commands. */ __redisShiftCallback(&ac->sub.invalid,dstcb); } return REDIS_OK; +oom: + __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; +} + +#define redisIsSpontaneousPushReply(r) \ + (redisIsPushReply(r) && !redisIsSubscribeReply(r)) + +static int redisIsSubscribeReply(redisReply *reply) { + char *str; + size_t len, off; + + /* We will always have at least one string with the subscribe/message type */ + if (reply->elements < 1 || reply->element[0]->type != REDIS_REPLY_STRING || + reply->element[0]->len < sizeof("message") - 1) + { + return 0; + } + + /* Get the string/len moving past 'p' if needed */ + off = tolower(reply->element[0]->str[0]) == 'p'; + str = reply->element[0]->str + off; + len = reply->element[0]->len - off; + + return !strncasecmp(str, "subscribe", len) || + !strncasecmp(str, "message", len); + } void redisProcessCallbacks(redisAsyncContext *ac) { @@ -427,7 +511,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) { if (reply == NULL) { /* When the connection is being disconnected and there are * no more replies, this is the cue to really disconnect. */ - if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0 + if (c->flags & REDIS_DISCONNECTING && hi_sdslen(c->obuf) == 0 && ac->replies.head == NULL) { __redisAsyncDisconnect(ac); return; @@ -443,8 +527,18 @@ void redisProcessCallbacks(redisAsyncContext *ac) { break; } - /* Even if the context is subscribed, pending regular callbacks will - * get a reply before pub/sub messages arrive. */ + /* Send any non-subscribe related PUSH messages to our PUSH handler + * while allowing subscribe related PUSH messages to pass through. + * This allows existing code to be backward compatible and work in + * either RESP2 or RESP3 mode. */ + if (redisIsSpontaneousPushReply(reply)) { + __redisRunPushCallback(ac, reply); + c->reader->fn->freeObject(reply); + continue; + } + + /* Even if the context is subscribed, pending regular + * callbacks will get a reply before pub/sub messages arrive. */ if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) { /* * A spontaneous reply in a not-subscribed context can be the error @@ -497,20 +591,31 @@ void redisProcessCallbacks(redisAsyncContext *ac) { __redisAsyncDisconnect(ac); } +static void __redisAsyncHandleConnectFailure(redisAsyncContext *ac) { + if (ac->onConnect) ac->onConnect(ac, REDIS_ERR); + __redisAsyncDisconnect(ac); +} + /* Internal helper function to detect socket status the first time a read or * write event fires. When connecting was not successful, the connect callback * is called with a REDIS_ERR status and the context is free'd. */ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { int completed = 0; redisContext *c = &(ac->c); + if (redisCheckConnectDone(c, &completed) == REDIS_ERR) { /* Error! */ redisCheckSocketError(c); - if (ac->onConnect) ac->onConnect(ac, REDIS_ERR); - __redisAsyncDisconnect(ac); + __redisAsyncHandleConnectFailure(ac); return REDIS_ERR; } else if (completed == 1) { /* connected! */ + if (c->connection_type == REDIS_CONN_TCP && + redisSetTcpNoDelay(c) == REDIS_ERR) { + __redisAsyncHandleConnectFailure(ac); + return REDIS_ERR; + } + if (ac->onConnect) ac->onConnect(ac, REDIS_OK); c->flags |= REDIS_CONNECTED; return REDIS_OK; @@ -582,8 +687,6 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { c->funcs->async_write(ac); } -void __redisSetError(redisContext *c, int type, const char *str); - void redisAsyncHandleTimeout(redisAsyncContext *ac) { redisContext *c = &(ac->c); redisCallback cb; @@ -641,7 +744,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void const char *cstr, *astr; size_t clen, alen; const char *p; - sds sname; + hisds sname; int ret; /* Don't accept new commands when the connection is about to be closed. */ @@ -665,7 +768,10 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void /* Add every channel/pattern to the list of subscription callbacks. */ while ((p = nextArgument(p,&astr,&alen)) != NULL) { - sname = sdsnewlen(astr,alen); + sname = hi_sdsnewlen(astr,alen); + if (sname == NULL) + goto oom; + if (pvariant) cbdict = ac->sub.patterns; else @@ -680,7 +786,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void ret = dictReplace(cbdict,sname,&cb); - if (ret == 0) sdsfree(sname); + if (ret == 0) hi_sdsfree(sname); } } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) { /* It is only useful to call (P)UNSUBSCRIBE when the context is @@ -709,6 +815,9 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void _EL_ADD_WRITE(ac); return REDIS_OK; +oom: + __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) { @@ -722,7 +831,7 @@ int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdat return REDIS_ERR; status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - free(cmd); + hi_free(cmd); return status; } @@ -736,14 +845,14 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata } int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd; int len; int status; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); if (len < 0) return REDIS_ERR; status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - sdsfree(cmd); + hi_sdsfree(cmd); return status; } @@ -752,15 +861,27 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void return status; } -void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { - if (!ac->c.timeout) { - ac->c.timeout = calloc(1, sizeof(tv)); - } - - if (tv.tv_sec == ac->c.timeout->tv_sec && - tv.tv_usec == ac->c.timeout->tv_usec) { - return; - } - - *ac->c.timeout = tv; +redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn) { + redisAsyncPushFn *old = ac->push_cb; + ac->push_cb = fn; + return old; +} + +int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.command_timeout) { + ac->c.command_timeout = hi_calloc(1, sizeof(tv)); + if (ac->c.command_timeout == NULL) { + __redisSetError(&ac->c, REDIS_ERR_OOM, "Out of memory"); + __redisAsyncCopyError(ac); + return REDIS_ERR; + } + } + + if (tv.tv_sec != ac->c.command_timeout->tv_sec || + tv.tv_usec != ac->c.command_timeout->tv_usec) + { + *ac->c.command_timeout = tv; + } + + return REDIS_OK; } diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 4f6b3b783..b1d2cb263 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -70,6 +70,7 @@ typedef struct redisAsyncContext { /* Not used by hiredis */ void *data; + void (*dataCleanup)(void *privdata); /* Event library data and hooks */ struct { @@ -105,6 +106,9 @@ typedef struct redisAsyncContext { struct dict *channels; struct dict *patterns; } sub; + + /* Any configured RESP3 PUSH handler */ + redisAsyncPushFn *push_cb; } redisAsyncContext; /* Functions that proxy to hiredis */ @@ -117,7 +121,8 @@ redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); -void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); +redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn); +int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h index d0133ae18..ea0558d42 100644 --- a/deps/hiredis/async_private.h +++ b/deps/hiredis/async_private.h @@ -51,18 +51,21 @@ #define _EL_CLEANUP(ctx) do { \ if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ ctx->ev.cleanup = NULL; \ - } while(0); + } while(0) static inline void refreshTimeout(redisAsyncContext *ctx) { - if (ctx->c.timeout && ctx->ev.scheduleTimer && - (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { - ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); - // } else { - // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); - // if (ctx->c.timeout){ - // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, - // ctx->c.timeout->tv_usec); - // } + #define REDIS_TIMER_ISSET(tvp) \ + (tvp && ((tvp)->tv_sec || (tvp)->tv_usec)) + + #define REDIS_EL_TIMER(ac, tvp) \ + if ((ac)->ev.scheduleTimer && REDIS_TIMER_ISSET(tvp)) { \ + (ac)->ev.scheduleTimer((ac)->ev.data, *(tvp)); \ + } + + if (ctx->c.flags & REDIS_CONNECTED) { + REDIS_EL_TIMER(ctx, ctx->c.command_timeout); + } else { + REDIS_EL_TIMER(ctx, ctx->c.connect_timeout); } } diff --git a/deps/hiredis/dict.c b/deps/hiredis/dict.c index e17a62546..34a33ead9 100644 --- a/deps/hiredis/dict.c +++ b/deps/hiredis/dict.c @@ -34,6 +34,7 @@ */ #include "fmacros.h" +#include "alloc.h" #include #include #include @@ -71,7 +72,10 @@ static void _dictReset(dict *ht) { /* Create a new hash table */ static dict *dictCreate(dictType *type, void *privDataPtr) { - dict *ht = malloc(sizeof(*ht)); + dict *ht = hi_malloc(sizeof(*ht)); + if (ht == NULL) + return NULL; + _dictInit(ht,type,privDataPtr); return ht; } @@ -97,7 +101,9 @@ static int dictExpand(dict *ht, unsigned long size) { _dictInit(&n, ht->type, ht->privdata); n.size = realsize; n.sizemask = realsize-1; - n.table = calloc(realsize,sizeof(dictEntry*)); + n.table = hi_calloc(realsize,sizeof(dictEntry*)); + if (n.table == NULL) + return DICT_ERR; /* Copy all the elements from the old to the new table: * note that if the old hash table is empty ht->size is zero, @@ -124,7 +130,7 @@ static int dictExpand(dict *ht, unsigned long size) { } } assert(ht->used == 0); - free(ht->table); + hi_free(ht->table); /* Remap the new hashtable in the old */ *ht = n; @@ -142,7 +148,10 @@ static int dictAdd(dict *ht, void *key, void *val) { return DICT_ERR; /* Allocates the memory and stores key */ - entry = malloc(sizeof(*entry)); + entry = hi_malloc(sizeof(*entry)); + if (entry == NULL) + return DICT_ERR; + entry->next = ht->table[index]; ht->table[index] = entry; @@ -166,6 +175,9 @@ static int dictReplace(dict *ht, void *key, void *val) { return 1; /* It already exists, get the entry */ entry = dictFind(ht, key); + if (entry == NULL) + return 0; + /* Free the old value and set the new one */ /* Set the new value and free the old one. Note that it is important * to do that in this order, as the value may just be exactly the same @@ -199,7 +211,7 @@ static int dictDelete(dict *ht, const void *key) { dictFreeEntryKey(ht,de); dictFreeEntryVal(ht,de); - free(de); + hi_free(de); ht->used--; return DICT_OK; } @@ -222,13 +234,13 @@ static int _dictClear(dict *ht) { nextHe = he->next; dictFreeEntryKey(ht, he); dictFreeEntryVal(ht, he); - free(he); + hi_free(he); ht->used--; he = nextHe; } } /* Free the table and the allocated cache structure */ - free(ht->table); + hi_free(ht->table); /* Re-initialize the table */ _dictReset(ht); return DICT_OK; /* never fails */ @@ -237,7 +249,7 @@ static int _dictClear(dict *ht) { /* Clear & Release the hash table */ static void dictRelease(dict *ht) { _dictClear(ht); - free(ht); + hi_free(ht); } static dictEntry *dictFind(dict *ht, const void *key) { @@ -256,7 +268,9 @@ static dictEntry *dictFind(dict *ht, const void *key) { } static dictIterator *dictGetIterator(dict *ht) { - dictIterator *iter = malloc(sizeof(*iter)); + dictIterator *iter = hi_malloc(sizeof(*iter)); + if (iter == NULL) + return NULL; iter->ht = ht; iter->index = -1; @@ -286,7 +300,7 @@ static dictEntry *dictNext(dictIterator *iter) { } static void dictReleaseIterator(dictIterator *iter) { - free(iter); + hi_free(iter); } /* ------------------------- private functions ------------------------------ */ @@ -294,7 +308,7 @@ static void dictReleaseIterator(dictIterator *iter) { /* Expand the hash table if needed */ static int _dictExpandIfNeeded(dict *ht) { /* If the hash table is empty expand it to the initial size, - * if the table is "full" dobule its size. */ + * if the table is "full" double its size. */ if (ht->size == 0) return dictExpand(ht, DICT_HT_INITIAL_SIZE); if (ht->used == ht->size) diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt index dd3a313ac..1d5bc56e0 100644 --- a/deps/hiredis/examples/CMakeLists.txt +++ b/deps/hiredis/examples/CMakeLists.txt @@ -44,3 +44,6 @@ ENDIF() ADD_EXECUTABLE(example example.c) TARGET_LINK_LIBRARIES(example hiredis) + +ADD_EXECUTABLE(example-push example-push.c) +TARGET_LINK_LIBRARIES(example-push hiredis) diff --git a/deps/hiredis/examples/example-ivykis.c b/deps/hiredis/examples/example-ivykis.c index 67affcef3..f57dc3887 100644 --- a/deps/hiredis/examples/example-ivykis.c +++ b/deps/hiredis/examples/example-ivykis.c @@ -33,7 +33,9 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif iv_init(); diff --git a/deps/hiredis/examples/example-libev.c b/deps/hiredis/examples/example-libev.c index cc8b166ec..ec474306b 100644 --- a/deps/hiredis/examples/example-libev.c +++ b/deps/hiredis/examples/example-libev.c @@ -33,7 +33,9 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); if (c->err) { diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c index 1021113b9..7d99af1ba 100644 --- a/deps/hiredis/examples/example-libevent-ssl.c +++ b/deps/hiredis/examples/example-libevent-ssl.c @@ -34,7 +34,10 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + struct event_base *base = event_base_new(); if (argc < 5) { fprintf(stderr, @@ -52,13 +55,25 @@ int main (int argc, char **argv) { const char *certKey = argv[5]; const char *caCert = argc > 5 ? argv[6] : NULL; + redisSSLContext *ssl; + redisSSLContextError ssl_error; + + redisInitOpenSSL(); + + ssl = redisCreateSSLContext(caCert, NULL, + cert, certKey, NULL, &ssl_error); + if (!ssl) { + printf("Error: %s\n", redisSSLContextGetError(ssl_error)); + return 1; + } + redisAsyncContext *c = redisAsyncConnect(hostname, port); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); return 1; } - if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + if (redisInitiateSSLWithContext(&c->c, ssl) != REDIS_OK) { printf("SSL Error!\n"); exit(1); } @@ -69,5 +84,7 @@ int main (int argc, char **argv) { redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); event_base_dispatch(base); + + redisFreeSSLContext(ssl); return 0; } diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c index 1fe71ae4e..49bddd0c2 100644 --- a/deps/hiredis/examples/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -38,13 +38,16 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + struct event_base *base = event_base_new(); redisOptions options = {0}; REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); struct timeval tv = {0}; tv.tv_sec = 1; - options.timeout = &tv; + options.connect_timeout = &tv; redisAsyncContext *c = redisAsyncConnectWithOptions(&options); diff --git a/deps/hiredis/examples/example-libuv.c b/deps/hiredis/examples/example-libuv.c index a5462d410..cbde452b9 100644 --- a/deps/hiredis/examples/example-libuv.c +++ b/deps/hiredis/examples/example-libuv.c @@ -33,7 +33,10 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + uv_loop_t* loop = uv_default_loop(); redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); diff --git a/deps/hiredis/examples/example-push.c b/deps/hiredis/examples/example-push.c new file mode 100644 index 000000000..2d4ab4dc0 --- /dev/null +++ b/deps/hiredis/examples/example-push.c @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include + +#define KEY_COUNT 5 + +#define panicAbort(fmt, ...) \ + do { \ + fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \ + exit(-1); \ + } while (0) + +static void assertReplyAndFree(redisContext *context, redisReply *reply, int type) { + if (reply == NULL) + panicAbort("NULL reply from server (error: %s)", context->errstr); + + if (reply->type != type) { + if (reply->type == REDIS_REPLY_ERROR) + fprintf(stderr, "Redis Error: %s\n", reply->str); + + panicAbort("Expected reply type %d but got type %d", type, reply->type); + } + + freeReplyObject(reply); +} + +/* Switch to the RESP3 protocol and enable client tracking */ +static void enableClientTracking(redisContext *c) { + redisReply *reply = redisCommand(c, "HELLO 3"); + if (reply == NULL || c->err) { + panicAbort("NULL reply or server error (error: %s)", c->errstr); + } + + if (reply->type != REDIS_REPLY_MAP) { + fprintf(stderr, "Error: Can't send HELLO 3 command. Are you sure you're "); + fprintf(stderr, "connected to redis-server >= 6.0.0?\nRedis error: %s\n", + reply->type == REDIS_REPLY_ERROR ? reply->str : "(unknown)"); + exit(-1); + } + + freeReplyObject(reply); + + /* Enable client tracking */ + reply = redisCommand(c, "CLIENT TRACKING ON"); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); +} + +void pushReplyHandler(void *privdata, void *r) { + redisReply *reply = r; + int *invalidations = privdata; + + /* Sanity check on the invalidation reply */ + if (reply->type != REDIS_REPLY_PUSH || reply->elements != 2 || + reply->element[1]->type != REDIS_REPLY_ARRAY || + reply->element[1]->element[0]->type != REDIS_REPLY_STRING) + { + panicAbort("%s", "Can't parse PUSH message!"); + } + + /* Increment our invalidation count */ + *invalidations += 1; + + printf("pushReplyHandler(): INVALIDATE '%s' (invalidation count: %d)\n", + reply->element[1]->element[0]->str, *invalidations); + + freeReplyObject(reply); +} + +/* We aren't actually freeing anything here, but it is included to show that we can + * have hiredis call our data destructor when freeing the context */ +void privdata_dtor(void *privdata) { + unsigned int *icount = privdata; + printf("privdata_dtor(): In context privdata dtor (invalidations: %u)\n", *icount); +} + +int main(int argc, char **argv) { + unsigned int j, invalidations = 0; + redisContext *c; + redisReply *reply; + + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = (argc > 2) ? atoi(argv[2]) : 6379; + + redisOptions o = {0}; + REDIS_OPTIONS_SET_TCP(&o, hostname, port); + + /* Set our context privdata to the address of our invalidation counter. Each + * time our PUSH handler is called, hiredis will pass the privdata for context. + * + * This could also be done after we create the context like so: + * + * c->privdata = &invalidations; + * c->free_privdata = privdata_dtor; + */ + REDIS_OPTIONS_SET_PRIVDATA(&o, &invalidations, privdata_dtor); + + /* Set our custom PUSH message handler */ + o.push_cb = pushReplyHandler; + + c = redisConnectWithOptions(&o); + if (c == NULL || c->err) + panicAbort("Connection error: %s", c ? c->errstr : "OOM"); + + /* Enable RESP3 and turn on client tracking */ + enableClientTracking(c); + + /* Set some keys and then read them back. Once we do that, Redis will deliver + * invalidation push messages whenever the key is modified */ + for (j = 0; j < KEY_COUNT; j++) { + reply = redisCommand(c, "SET key:%d initial:%d", j, j); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); + + reply = redisCommand(c, "GET key:%d", j); + assertReplyAndFree(c, reply, REDIS_REPLY_STRING); + } + + /* Trigger invalidation messages by updating keys we just read */ + for (j = 0; j < KEY_COUNT; j++) { + printf(" main(): SET key:%d update:%d\n", j, j); + reply = redisCommand(c, "SET key:%d update:%d", j, j); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); + printf(" main(): SET REPLY OK\n"); + } + + printf("\nTotal detected invalidations: %d, expected: %d\n", invalidations, KEY_COUNT); + + /* PING server */ + redisFree(c); +} diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c index 81f4648c6..c754177cf 100644 --- a/deps/hiredis/examples/example-ssl.c +++ b/deps/hiredis/examples/example-ssl.c @@ -4,9 +4,12 @@ #include #include +#include int main(int argc, char **argv) { unsigned int j; + redisSSLContext *ssl; + redisSSLContextError ssl_error; redisContext *c; redisReply *reply; if (argc < 4) { @@ -19,10 +22,18 @@ int main(int argc, char **argv) { const char *key = argv[4]; const char *ca = argc > 4 ? argv[5] : NULL; + redisInitOpenSSL(); + ssl = redisCreateSSLContext(ca, NULL, cert, key, NULL, &ssl_error); + if (!ssl) { + printf("SSL Context error: %s\n", + redisSSLContextGetError(ssl_error)); + exit(1); + } + struct timeval tv = { 1, 500000 }; // 1.5 seconds redisOptions options = {0}; REDIS_OPTIONS_SET_TCP(&options, hostname, port); - options.timeout = &tv; + options.connect_timeout = &tv; c = redisConnectWithOptions(&options); if (c == NULL || c->err) { @@ -35,7 +46,7 @@ int main(int argc, char **argv) { exit(1); } - if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) { printf("Couldn't initialize SSL!\n"); printf("Error: %s\n", c->errstr); redisFree(c); @@ -93,5 +104,7 @@ int main(int argc, char **argv) { /* Disconnects and frees the context */ redisFree(c); + redisFreeSSLContext(ssl); + return 0; } diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 0e93fc8b3..15dacbd18 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -1,8 +1,8 @@ #include #include #include - #include +#include int main(int argc, char **argv) { unsigned int j, isunix = 0; diff --git a/deps/hiredis/hiredis-config.cmake.in b/deps/hiredis/hiredis-config.cmake.in new file mode 100644 index 000000000..98851dcee --- /dev/null +++ b/deps/hiredis/hiredis-config.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +set_and_check(hiredis_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@") + +IF (NOT TARGET hiredis::hiredis) + INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis-targets.cmake) +ENDIF() + +SET(hiredis_LIBRARIES hiredis::hiredis) +SET(hiredis_INCLUDE_DIRS ${hiredis_INCLUDEDIR}) + +check_required_components(hiredis) + diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index abd94c01d..51f22a665 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -44,8 +44,11 @@ #include "async.h" #include "win32.h" +extern int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout); +extern int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout); + static redisContextFuncs redisContextDefaultFuncs = { - .free_privdata = NULL, + .free_privctx = NULL, .async_read = redisAsyncRead, .async_write = redisAsyncWrite, .read = redisNetRead, @@ -74,7 +77,7 @@ static redisReplyObjectFunctions defaultFunctions = { /* Create a reply object */ static redisReply *createReplyObject(int type) { - redisReply *r = calloc(1,sizeof(*r)); + redisReply *r = hi_calloc(1,sizeof(*r)); if (r == NULL) return NULL; @@ -97,20 +100,22 @@ void freeReplyObject(void *reply) { case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: case REDIS_REPLY_SET: + case REDIS_REPLY_PUSH: if (r->element != NULL) { for (j = 0; j < r->elements; j++) freeReplyObject(r->element[j]); - free(r->element); + hi_free(r->element); } break; case REDIS_REPLY_ERROR: case REDIS_REPLY_STATUS: case REDIS_REPLY_STRING: case REDIS_REPLY_DOUBLE: - free(r->str); + case REDIS_REPLY_VERB: + hi_free(r->str); break; } - free(r); + hi_free(r); } static void *createStringObject(const redisReadTask *task, char *str, size_t len) { @@ -128,22 +133,18 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len /* Copy string value */ if (task->type == REDIS_REPLY_VERB) { - buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } + buf = hi_malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) goto oom; + memcpy(r->vtype,str,3); r->vtype[3] = '\0'; memcpy(buf,str+4,len-4); buf[len-4] = '\0'; - r->len = len-4; + r->len = len - 4; } else { - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } + buf = hi_malloc(len+1); + if (buf == NULL) goto oom; + memcpy(buf,str,len); buf[len] = '\0'; r->len = len; @@ -154,10 +155,15 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; + +oom: + freeReplyObject(r); + return NULL; } static void *createArrayObject(const redisReadTask *task, size_t elements) { @@ -168,7 +174,7 @@ static void *createArrayObject(const redisReadTask *task, size_t elements) { return NULL; if (elements > 0) { - r->element = calloc(elements,sizeof(redisReply*)); + r->element = hi_calloc(elements,sizeof(redisReply*)); if (r->element == NULL) { freeReplyObject(r); return NULL; @@ -181,7 +187,8 @@ static void *createArrayObject(const redisReadTask *task, size_t elements) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -200,7 +207,8 @@ static void *createIntegerObject(const redisReadTask *task, long long value) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -214,7 +222,7 @@ static void *createDoubleObject(const redisReadTask *task, double value, char *s return NULL; r->dval = value; - r->str = malloc(len+1); + r->str = hi_malloc(len+1); if (r->str == NULL) { freeReplyObject(r); return NULL; @@ -249,7 +257,8 @@ static void *createNilObject(const redisReadTask *task) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -297,7 +306,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { const char *c = format; char *cmd = NULL; /* final command */ int pos; /* position in final command */ - sds curarg, newarg; /* current argument */ + hisds curarg, newarg; /* current argument */ int touched = 0; /* was the current argument touched? */ char **curargv = NULL, **newargv = NULL; int argc = 0; @@ -310,7 +319,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { return -1; /* Build the command string accordingly to protocol */ - curarg = sdsempty(); + curarg = hi_sdsempty(); if (curarg == NULL) return -1; @@ -318,19 +327,19 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { if (*c != '%' || c[1] == '\0') { if (*c == ' ') { if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); + newargv = hi_realloc(curargv,sizeof(char*)*(argc+1)); if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); + totlen += bulklen(hi_sdslen(curarg)); /* curarg is put in argv so it can be overwritten. */ - curarg = sdsempty(); + curarg = hi_sdsempty(); if (curarg == NULL) goto memory_err; touched = 0; } } else { - newarg = sdscatlen(curarg,c,1); + newarg = hi_sdscatlen(curarg,c,1); if (newarg == NULL) goto memory_err; curarg = newarg; touched = 1; @@ -347,16 +356,16 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { arg = va_arg(ap,char*); size = strlen(arg); if (size > 0) - newarg = sdscatlen(curarg,arg,size); + newarg = hi_sdscatlen(curarg,arg,size); break; case 'b': arg = va_arg(ap,char*); size = va_arg(ap,size_t); if (size > 0) - newarg = sdscatlen(curarg,arg,size); + newarg = hi_sdscatlen(curarg,arg,size); break; case '%': - newarg = sdscat(curarg,"%"); + newarg = hi_sdscat(curarg,"%"); break; default: /* Try to detect printf format */ @@ -444,7 +453,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { if (_l < sizeof(_format)-2) { memcpy(_format,c,_l); _format[_l] = '\0'; - newarg = sdscatvprintf(curarg,_format,_cpy); + newarg = hi_sdscatvprintf(curarg,_format,_cpy); /* Update current position (note: outer blocks * increment c twice so compensate here) */ @@ -467,13 +476,13 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { /* Add the last argument if needed */ if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); + newargv = hi_realloc(curargv,sizeof(char*)*(argc+1)); if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); + totlen += bulklen(hi_sdslen(curarg)); } else { - sdsfree(curarg); + hi_sdsfree(curarg); } /* Clear curarg because it was put in curargv or was free'd. */ @@ -483,22 +492,22 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { totlen += 1+countDigits(argc)+2; /* Build the command at protocol level */ - cmd = malloc(totlen+1); + cmd = hi_malloc(totlen+1); if (cmd == NULL) goto memory_err; pos = sprintf(cmd,"*%d\r\n",argc); for (j = 0; j < argc; j++) { - pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j])); - memcpy(cmd+pos,curargv[j],sdslen(curargv[j])); - pos += sdslen(curargv[j]); - sdsfree(curargv[j]); + pos += sprintf(cmd+pos,"$%zu\r\n",hi_sdslen(curargv[j])); + memcpy(cmd+pos,curargv[j],hi_sdslen(curargv[j])); + pos += hi_sdslen(curargv[j]); + hi_sdsfree(curargv[j]); cmd[pos++] = '\r'; cmd[pos++] = '\n'; } assert(pos == totlen); cmd[pos] = '\0'; - free(curargv); + hi_free(curargv); *target = cmd; return totlen; @@ -513,12 +522,12 @@ memory_err: cleanup: if (curargv) { while(argc--) - sdsfree(curargv[argc]); - free(curargv); + hi_sdsfree(curargv[argc]); + hi_free(curargv); } - sdsfree(curarg); - free(cmd); + hi_sdsfree(curarg); + hi_free(cmd); return error_type; } @@ -550,16 +559,16 @@ int redisFormatCommand(char **target, const char *format, ...) { return len; } -/* Format a command according to the Redis protocol using an sds string and - * sdscatfmt for the processing of arguments. This function takes the +/* Format a command according to the Redis protocol using an hisds string and + * hi_sdscatfmt for the processing of arguments. This function takes the * number of arguments, an array with arguments and an array with their * lengths. If the latter is set to NULL, strlen will be used to compute the * argument lengths. */ -int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, +int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd, aux; unsigned long long totlen; int j; size_t len; @@ -576,32 +585,36 @@ int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, } /* Use an SDS string for command construction */ - cmd = sdsempty(); + cmd = hi_sdsempty(); if (cmd == NULL) return -1; /* We already know how much storage we need */ - cmd = sdsMakeRoomFor(cmd, totlen); - if (cmd == NULL) + aux = hi_sdsMakeRoomFor(cmd, totlen); + if (aux == NULL) { + hi_sdsfree(cmd); return -1; - - /* Construct command */ - cmd = sdscatfmt(cmd, "*%i\r\n", argc); - for (j=0; j < argc; j++) { - len = argvlen ? argvlen[j] : strlen(argv[j]); - cmd = sdscatfmt(cmd, "$%u\r\n", len); - cmd = sdscatlen(cmd, argv[j], len); - cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); } - assert(sdslen(cmd)==totlen); + cmd = aux; + + /* Construct command */ + cmd = hi_sdscatfmt(cmd, "*%i\r\n", argc); + for (j=0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + cmd = hi_sdscatfmt(cmd, "$%u\r\n", len); + cmd = hi_sdscatlen(cmd, argv[j], len); + cmd = hi_sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); + } + + assert(hi_sdslen(cmd)==totlen); *target = cmd; return totlen; } -void redisFreeSdsCommand(sds cmd) { - sdsfree(cmd); +void redisFreeSdsCommand(hisds cmd) { + hi_sdsfree(cmd); } /* Format a command according to the Redis protocol. This function takes the @@ -627,7 +640,7 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz } /* Build the command at protocol level */ - cmd = malloc(totlen+1); + cmd = hi_malloc(totlen+1); if (cmd == NULL) return -1; @@ -648,7 +661,7 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz } void redisFreeCommand(char *cmd) { - free(cmd); + hi_free(cmd); } void __redisSetError(redisContext *c, int type, const char *str) { @@ -671,15 +684,21 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(const redisOptions *options) { +static void redisPushAutoFree(void *privdata, void *reply) { + (void)privdata; + freeReplyObject(reply); +} + +static redisContext *redisContextInit(void) { redisContext *c; - c = calloc(1, sizeof(*c)); + c = hi_calloc(1, sizeof(*c)); if (c == NULL) return NULL; c->funcs = &redisContextDefaultFuncs; - c->obuf = sdsempty(); + + c->obuf = hi_sdsempty(); c->reader = redisReaderCreate(); c->fd = REDIS_INVALID_FD; @@ -687,7 +706,7 @@ static redisContext *redisContextInit(const redisOptions *options) { redisFree(c); return NULL; } - (void)options; /* options are used in other functions */ + return c; } @@ -696,18 +715,23 @@ void redisFree(redisContext *c) { return; redisNetClose(c); - sdsfree(c->obuf); + hi_sdsfree(c->obuf); redisReaderFree(c->reader); - free(c->tcp.host); - free(c->tcp.source_addr); - free(c->unix_sock.path); - free(c->timeout); - free(c->saddr); - if (c->funcs->free_privdata) { - c->funcs->free_privdata(c->privdata); - } + hi_free(c->tcp.host); + hi_free(c->tcp.source_addr); + hi_free(c->unix_sock.path); + hi_free(c->connect_timeout); + hi_free(c->command_timeout); + hi_free(c->saddr); + + if (c->privdata && c->free_privdata) + c->free_privdata(c->privdata); + + if (c->funcs->free_privctx) + c->funcs->free_privctx(c->privctx); + memset(c, 0xff, sizeof(*c)); - free(c); + hi_free(c); } redisFD redisFreeKeepFd(redisContext *c) { @@ -721,35 +745,46 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->privdata && c->funcs->free_privdata) { - c->funcs->free_privdata(c->privdata); - c->privdata = NULL; + if (c->privctx && c->funcs->free_privctx) { + c->funcs->free_privctx(c->privctx); + c->privctx = NULL; } redisNetClose(c); - sdsfree(c->obuf); + hi_sdsfree(c->obuf); redisReaderFree(c->reader); - c->obuf = sdsempty(); + c->obuf = hi_sdsempty(); c->reader = redisReaderCreate(); + if (c->obuf == NULL || c->reader == NULL) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; + } + + int ret = REDIS_ERR; if (c->connection_type == REDIS_CONN_TCP) { - return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, - c->timeout, c->tcp.source_addr); + ret = redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, + c->connect_timeout, c->tcp.source_addr); } else if (c->connection_type == REDIS_CONN_UNIX) { - return redisContextConnectUnix(c, c->unix_sock.path, c->timeout); + ret = redisContextConnectUnix(c, c->unix_sock.path, c->connect_timeout); } else { /* Something bad happened here and shouldn't have. There isn't enough information in the context to reconnect. */ __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect"); + ret = REDIS_ERR; } - return REDIS_ERR; + if (c->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *c->command_timeout); + } + + return ret; } redisContext *redisConnectWithOptions(const redisOptions *options) { - redisContext *c = redisContextInit(options); + redisContext *c = redisContextInit(); if (c == NULL) { return NULL; } @@ -760,16 +795,32 @@ redisContext *redisConnectWithOptions(const redisOptions *options) { c->flags |= REDIS_REUSEADDR; } if (options->options & REDIS_OPT_NOAUTOFREE) { - c->flags |= REDIS_NO_AUTO_FREE; + c->flags |= REDIS_NO_AUTO_FREE; + } + + /* Set any user supplied RESP3 PUSH handler or use freeReplyObject + * as a default unless specifically flagged that we don't want one. */ + if (options->push_cb != NULL) + redisSetPushCallback(c, options->push_cb); + else if (!(options->options & REDIS_OPT_NO_PUSH_AUTOFREE)) + redisSetPushCallback(c, redisPushAutoFree); + + c->privdata = options->privdata; + c->free_privdata = options->free_privdata; + + if (redisContextUpdateConnectTimeout(c, options->connect_timeout) != REDIS_OK || + redisContextUpdateCommandTimeout(c, options->command_timeout) != REDIS_OK) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return c; } if (options->type == REDIS_CONN_TCP) { redisContextConnectBindTcp(c, options->endpoint.tcp.ip, - options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.port, options->connect_timeout, options->endpoint.tcp.source_addr); } else if (options->type == REDIS_CONN_UNIX) { redisContextConnectUnix(c, options->endpoint.unix_socket, - options->timeout); + options->connect_timeout); } else if (options->type == REDIS_CONN_USERFD) { c->fd = options->endpoint.fd; c->flags |= REDIS_CONNECTED; @@ -777,9 +828,11 @@ redisContext *redisConnectWithOptions(const redisOptions *options) { // Unknown type - FIXME - FREE return NULL; } - if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { - redisContextSetTimeout(c, *options->timeout); + + if (options->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->command_timeout); } + return c; } @@ -795,7 +848,7 @@ redisContext *redisConnect(const char *ip, int port) { redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { redisOptions options = {0}; REDIS_OPTIONS_SET_TCP(&options, ip, port); - options.timeout = &tv; + options.connect_timeout = &tv; return redisConnectWithOptions(&options); } @@ -833,7 +886,7 @@ redisContext *redisConnectUnix(const char *path) { redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { redisOptions options = {0}; REDIS_OPTIONS_SET_UNIX(&options, path); - options.timeout = &tv; + options.connect_timeout = &tv; return redisConnectWithOptions(&options); } @@ -865,6 +918,13 @@ int redisEnableKeepAlive(redisContext *c) { return REDIS_OK; } +/* Set a user provided RESP3 PUSH handler and return any old one set. */ +redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn) { + redisPushFn *old = c->push_cb; + c->push_cb = fn; + return old; +} + /* Use this function to handle a read event on the descriptor. It will try * and read some bytes from the socket and feed them to the reply parser. * @@ -906,21 +966,27 @@ int redisBufferWrite(redisContext *c, int *done) { if (c->err) return REDIS_ERR; - if (sdslen(c->obuf) > 0) { - int nwritten = c->funcs->write(c); + if (hi_sdslen(c->obuf) > 0) { + ssize_t nwritten = c->funcs->write(c); if (nwritten < 0) { return REDIS_ERR; } else if (nwritten > 0) { - if (nwritten == (signed)sdslen(c->obuf)) { - sdsfree(c->obuf); - c->obuf = sdsempty(); + if (nwritten == (ssize_t)hi_sdslen(c->obuf)) { + hi_sdsfree(c->obuf); + c->obuf = hi_sdsempty(); + if (c->obuf == NULL) + goto oom; } else { - sdsrange(c->obuf,nwritten,-1); + if (hi_sdsrange(c->obuf,nwritten,-1) < 0) goto oom; } } } - if (done != NULL) *done = (sdslen(c->obuf) == 0); + if (done != NULL) *done = (hi_sdslen(c->obuf) == 0); return REDIS_OK; + +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } /* Internal helper function to try and get a reply from the reader, @@ -930,9 +996,21 @@ int redisGetReplyFromReader(redisContext *c, void **reply) { __redisSetError(c,c->reader->err,c->reader->errstr); return REDIS_ERR; } + return REDIS_OK; } +/* Internal helper that returns 1 if the reply was a RESP3 PUSH + * message and we handled it with a user-provided callback. */ +static int redisHandledPushReply(redisContext *c, void *reply) { + if (reply && c->push_cb && redisIsPushReply(reply)) { + c->push_cb(c->privdata, reply); + return 1; + } + + return 0; +} + int redisGetReply(redisContext *c, void **reply) { int wdone = 0; void *aux = NULL; @@ -953,13 +1031,23 @@ int redisGetReply(redisContext *c, void **reply) { do { if (redisBufferRead(c) == REDIS_ERR) return REDIS_ERR; - if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) - return REDIS_ERR; + + /* We loop here in case the user has specified a RESP3 + * PUSH handler (e.g. for client tracking). */ + do { + if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) + return REDIS_ERR; + } while (redisHandledPushReply(c, aux)); } while (aux == NULL); } - /* Set reply object */ - if (reply != NULL) *reply = aux; + /* Set reply or free it if we were passed NULL */ + if (reply != NULL) { + *reply = aux; + } else { + freeReplyObject(aux); + } + return REDIS_OK; } @@ -971,9 +1059,9 @@ int redisGetReply(redisContext *c, void **reply) { * the reply (or replies in pub/sub). */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) { - sds newbuf; + hisds newbuf; - newbuf = sdscatlen(c->obuf,cmd,len); + newbuf = hi_sdscatlen(c->obuf,cmd,len); if (newbuf == NULL) { __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); return REDIS_ERR; @@ -1006,11 +1094,11 @@ int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - free(cmd); + hi_free(cmd); return REDIS_ERR; } - free(cmd); + hi_free(cmd); return REDIS_OK; } @@ -1025,7 +1113,7 @@ int redisAppendCommand(redisContext *c, const char *format, ...) { } int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd; int len; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); @@ -1035,11 +1123,11 @@ int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const s } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - sdsfree(cmd); + hi_sdsfree(cmd); return REDIS_ERR; } - sdsfree(cmd); + hi_sdsfree(cmd); return REDIS_OK; } diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index 69dc39c5e..b597394d4 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -39,14 +39,16 @@ #include /* for struct timeval */ #else struct timeval; /* forward declaration */ +typedef long long ssize_t; #endif #include /* uintXX_t, etc */ -#include "sds.h" /* for sds */ +#include "sds.h" /* for hisds */ +#include "alloc.h" /* for allocation wrappers */ -#define HIREDIS_MAJOR 0 -#define HIREDIS_MINOR 14 +#define HIREDIS_MAJOR 1 +#define HIREDIS_MINOR 0 #define HIREDIS_PATCH 0 -#define HIREDIS_SONAME 0.14 +#define HIREDIS_SONAME 1.0.0 /* Connection type can be blocking or non-blocking and is set in the * least significant bit of the flags field in redisContext. */ @@ -90,6 +92,15 @@ struct timeval; /* forward declaration */ * SO_REUSEADDR is being used. */ #define REDIS_CONNECT_RETRIES 10 +/* Forward declarations for structs defined elsewhere */ +struct redisAsyncContext; +struct redisContext; + +/* RESP3 push helpers and callback prototypes */ +#define redisIsPushReply(r) (((redisReply*)(r))->type == REDIS_REPLY_PUSH) +typedef void (redisPushFn)(void *, void *); +typedef void (redisAsyncPushFn)(struct redisAsyncContext *, void *); + #ifdef __cplusplus extern "C" { #endif @@ -101,7 +112,7 @@ typedef struct redisReply { double dval; /* The double when type is REDIS_REPLY_DOUBLE */ size_t len; /* Length of string */ char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING - and REDIS_REPLY_DOUBLE (in additionl to dval). */ + REDIS_REPLY_VERB, and REDIS_REPLY_DOUBLE (in additional to dval). */ char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ @@ -117,9 +128,9 @@ void freeReplyObject(void *reply); int redisvFormatCommand(char **target, const char *format, va_list ap); int redisFormatCommand(char **target, const char *format, ...); int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); -int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen); +int redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen); void redisFreeCommand(char *cmd); -void redisFreeSdsCommand(sds cmd); +void redisFreeSdsCommand(hisds cmd); enum redisConnectionType { REDIS_CONN_TCP, @@ -138,6 +149,9 @@ struct redisSsl; */ #define REDIS_OPT_NOAUTOFREE 0x04 +/* Don't automatically intercept and free RESP3 PUSH replies. */ +#define REDIS_OPT_NO_PUSH_AUTOFREE 0x08 + /* In Unix systems a file descriptor is a regular signed int, with -1 * representing an invalid descriptor. In Windows it is a SOCKET * (32- or 64-bit unsigned integer depending on the architecture), where @@ -162,8 +176,11 @@ typedef struct { int type; /* bit field of REDIS_OPT_xxx */ int options; - /* timeout value. if NULL, no timeout is used */ - const struct timeval *timeout; + /* timeout value for connect operation. If NULL, no timeout is used */ + const struct timeval *connect_timeout; + /* timeout value for commands. If NULL, no timeout is used. This can be + * updated at runtime with redisSetTimeout/redisAsyncSetTimeout. */ + const struct timeval *command_timeout; union { /** use this field for tcp/ip connections */ struct { @@ -178,6 +195,14 @@ typedef struct { * file descriptor */ redisFD fd; } endpoint; + + /* Optional user defined data/destructor */ + void *privdata; + void (*free_privdata)(void *); + + /* A user defined PUSH message callback */ + redisPushFn *push_cb; + redisAsyncPushFn *async_push_cb; } redisOptions; /** @@ -192,15 +217,16 @@ typedef struct { (opts)->type = REDIS_CONN_UNIX; \ (opts)->endpoint.unix_socket = path; -struct redisAsyncContext; -struct redisContext; +#define REDIS_OPTIONS_SET_PRIVDATA(opts, data, dtor) \ + (opts)->privdata = data; \ + (opts)->free_privdata = dtor; \ typedef struct redisContextFuncs { - void (*free_privdata)(void *); + void (*free_privctx)(void *); void (*async_read)(struct redisAsyncContext *); void (*async_write)(struct redisAsyncContext *); - int (*read)(struct redisContext *, char *, size_t); - int (*write)(struct redisContext *); + ssize_t (*read)(struct redisContext *, char *, size_t); + ssize_t (*write)(struct redisContext *); } redisContextFuncs; /* Context for a connection to Redis */ @@ -215,7 +241,8 @@ typedef struct redisContext { redisReader *reader; /* Protocol reader */ enum redisConnectionType connection_type; - struct timeval *timeout; + struct timeval *connect_timeout; + struct timeval *command_timeout; struct { char *host; @@ -231,8 +258,17 @@ typedef struct redisContext { struct sockadr *saddr; size_t addrlen; - /* Additional private data for hiredis addons such as SSL */ + /* Optional data and corresponding destructor users can use to provide + * context to a given redisContext. Not used by hiredis. */ void *privdata; + void (*free_privdata)(void *); + + /* Internal context pointer presently used by hiredis to manage + * SSL connections. */ + void *privctx; + + /* An optional RESP3 PUSH handler */ + redisPushFn *push_cb; } redisContext; redisContext *redisConnectWithOptions(const redisOptions *options); @@ -259,6 +295,7 @@ redisContext *redisConnectFd(redisFD fd); */ int redisReconnect(redisContext *c); +redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in index 140b040f1..91b773183 100644 --- a/deps/hiredis/hiredis.pc.in +++ b/deps/hiredis/hiredis.pc.in @@ -1,6 +1,7 @@ prefix=@CMAKE_INSTALL_PREFIX@ +install_libdir=@CMAKE_INSTALL_LIBDIR@ exec_prefix=${prefix} -libdir=${exec_prefix}/lib +libdir=${exec_prefix}/${install_libdir} includedir=${prefix}/include pkgincludedir=${includedir}/hiredis diff --git a/deps/hiredis/hiredis_ssl-config.cmake.in b/deps/hiredis/hiredis_ssl-config.cmake.in new file mode 100644 index 000000000..9a283dfc2 --- /dev/null +++ b/deps/hiredis/hiredis_ssl-config.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +set_and_check(hiredis_ssl_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@") + +IF (NOT TARGET hiredis::hiredis_ssl) + INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis_ssl-targets.cmake) +ENDIF() + +SET(hiredis_ssl_LIBRARIES hiredis::hiredis_ssl) +SET(hiredis_ssl_INCLUDE_DIRS ${hiredis_ssl_INCLUDEDIR}) + +check_required_components(hiredis_ssl) + diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h index f844f9548..604efe0c1 100644 --- a/deps/hiredis/hiredis_ssl.h +++ b/deps/hiredis/hiredis_ssl.h @@ -32,22 +32,96 @@ #ifndef __HIREDIS_SSL_H #define __HIREDIS_SSL_H +#ifdef __cplusplus +extern "C" { +#endif + /* This is the underlying struct for SSL in ssl.h, which is not included to * keep build dependencies short here. */ struct ssl_st; -/** - * Secure the connection using SSL. This should be done before any command is - * executed on the connection. +/* A wrapper around OpenSSL SSL_CTX to allow easy SSL use without directly + * calling OpenSSL. */ -int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, - const char *keypath, const char *servername); +typedef struct redisSSLContext redisSSLContext; /** - * Initiate SSL/TLS negotiation on a provided context. + * Initialization errors that redisCreateSSLContext() may return. + */ + +typedef enum { + REDIS_SSL_CTX_NONE = 0, /* No Error */ + REDIS_SSL_CTX_CREATE_FAILED, /* Failed to create OpenSSL SSL_CTX */ + REDIS_SSL_CTX_CERT_KEY_REQUIRED, /* Client cert and key must both be specified or skipped */ + REDIS_SSL_CTX_CA_CERT_LOAD_FAILED, /* Failed to load CA Certificate or CA Path */ + REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED, /* Failed to load client certificate */ + REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED /* Failed to load private key */ +} redisSSLContextError; + +/** + * Return the error message corresponding with the specified error code. + */ + +const char *redisSSLContextGetError(redisSSLContextError error); + +/** + * Helper function to initialize the OpenSSL library. + * + * OpenSSL requires one-time initialization before it can be used. Callers should + * call this function only once, and only if OpenSSL is not directly initialized + * elsewhere. + */ +int redisInitOpenSSL(void); + +/** + * Helper function to initialize an OpenSSL context that can be used + * to initiate SSL connections. + * + * cacert_filename is an optional name of a CA certificate/bundle file to load + * and use for validation. + * + * capath is an optional directory path where trusted CA certificate files are + * stored in an OpenSSL-compatible structure. + * + * cert_filename and private_key_filename are optional names of a client side + * certificate and private key files to use for authentication. They need to + * be both specified or omitted. + * + * server_name is an optional and will be used as a server name indication + * (SNI) TLS extension. + * + * If error is non-null, it will be populated in case the context creation fails + * (returning a NULL). + */ + +redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath, + const char *cert_filename, const char *private_key_filename, + const char *server_name, redisSSLContextError *error); + +/** + * Free a previously created OpenSSL context. + */ +void redisFreeSSLContext(redisSSLContext *redis_ssl_ctx); + +/** + * Initiate SSL on an existing redisContext. + * + * This is similar to redisInitiateSSL() but does not require the caller + * to directly interact with OpenSSL, and instead uses a redisSSLContext + * previously created using redisCreateSSLContext(). + */ + +int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx); + +/** + * Initiate SSL/TLS negotiation on a provided OpenSSL SSL object. */ int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); +#ifdef __cplusplus +} +#endif + #endif /* __HIREDIS_SSL_H */ diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index e5f40b0a4..88f9aff25 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -57,8 +57,8 @@ void redisNetClose(redisContext *c) { } } -int redisNetRead(redisContext *c, char *buf, size_t bufcap) { - int nread = recv(c->fd, buf, bufcap, 0); +ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap) { + ssize_t nread = recv(c->fd, buf, bufcap, 0); if (nread == -1) { if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ @@ -79,8 +79,8 @@ int redisNetRead(redisContext *c, char *buf, size_t bufcap) { } } -int redisNetWrite(redisContext *c) { - int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); +ssize_t redisNetWrite(redisContext *c) { + ssize_t nwritten = send(c->fd, c->obuf, hi_sdslen(c->obuf), 0); if (nwritten < 0) { if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ @@ -203,7 +203,7 @@ int redisKeepAlive(redisContext *c, int interval) { return REDIS_OK; } -static int redisSetTcpNoDelay(redisContext *c) { +int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); @@ -217,7 +217,7 @@ static int redisSetTcpNoDelay(redisContext *c) { static int redisContextTimeoutMsec(redisContext *c, long *result) { - const struct timeval *timeout = c->timeout; + const struct timeval *timeout = c->connect_timeout; long msec = -1; /* Only use timeout when not NULL. */ @@ -316,11 +316,7 @@ int redisCheckSocketError(redisContext *c) { int redisContextSetTimeout(redisContext *c, const struct timeval tv) { const void *to_ptr = &tv; size_t to_sz = sizeof(tv); -#ifdef _WIN32 - DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; - to_ptr = &timeout_msec; - to_sz = sizeof(timeout_msec); -#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; @@ -332,6 +328,38 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { return REDIS_OK; } +int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout) { + /* Same timeval struct, short circuit */ + if (c->connect_timeout == timeout) + return REDIS_OK; + + /* Allocate context timeval if we need to */ + if (c->connect_timeout == NULL) { + c->connect_timeout = hi_malloc(sizeof(*c->connect_timeout)); + if (c->connect_timeout == NULL) + return REDIS_ERR; + } + + memcpy(c->connect_timeout, timeout, sizeof(*c->connect_timeout)); + return REDIS_OK; +} + +int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout) { + /* Same timeval struct, short circuit */ + if (c->command_timeout == timeout) + return REDIS_OK; + + /* Allocate context timeval if we need to */ + if (c->command_timeout == NULL) { + c->command_timeout = hi_malloc(sizeof(*c->command_timeout)); + if (c->command_timeout == NULL) + return REDIS_ERR; + } + + memcpy(c->command_timeout, timeout, sizeof(*c->command_timeout)); + return REDIS_OK; +} + static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { @@ -356,21 +384,19 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, * This is a bit ugly, but atleast it works and doesn't leak memory. **/ if (c->tcp.host != addr) { - free(c->tcp.host); + hi_free(c->tcp.host); - c->tcp.host = strdup(addr); + c->tcp.host = hi_strdup(addr); + if (c->tcp.host == NULL) + goto oom; } if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); - - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } + if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR) + goto oom; } else { - free(c->timeout); - c->timeout = NULL; + hi_free(c->connect_timeout); + c->connect_timeout = NULL; } if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) { @@ -379,11 +405,11 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } if (source_addr == NULL) { - free(c->tcp.source_addr); + hi_free(c->tcp.source_addr); c->tcp.source_addr = NULL; } else if (c->tcp.source_addr != source_addr) { - free(c->tcp.source_addr); - c->tcp.source_addr = strdup(source_addr); + hi_free(c->tcp.source_addr); + c->tcp.source_addr = hi_strdup(source_addr); } snprintf(_port, 6, "%d", port); @@ -446,8 +472,11 @@ addrretry: } /* For repeat connection */ - free(c->saddr); - c->saddr = malloc(p->ai_addrlen); + hi_free(c->saddr); + c->saddr = hi_malloc(p->ai_addrlen); + if (c->saddr == NULL) + goto oom; + memcpy(c->saddr, p->ai_addr, p->ai_addrlen); c->addrlen = p->ai_addrlen; @@ -474,12 +503,12 @@ addrretry: wait_for_ready: if (redisContextWaitReady(c,timeout_msec) != REDIS_OK) goto error; + if (redisSetTcpNoDelay(c) != REDIS_OK) + goto error; } } if (blocking && redisSetBlocking(c,1) != REDIS_OK) goto error; - if (redisSetTcpNoDelay(c) != REDIS_OK) - goto error; c->flags |= REDIS_CONNECTED; rv = REDIS_OK; @@ -492,6 +521,8 @@ addrretry: goto error; } +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); error: rv = REDIS_ERR; end: @@ -525,25 +556,32 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time return REDIS_ERR; c->connection_type = REDIS_CONN_UNIX; - if (c->unix_sock.path != path) - c->unix_sock.path = strdup(path); + if (c->unix_sock.path != path) { + hi_free(c->unix_sock.path); + + c->unix_sock.path = hi_strdup(path); + if (c->unix_sock.path == NULL) + goto oom; + } if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); - - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } + if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR) + goto oom; } else { - free(c->timeout); - c->timeout = NULL; + hi_free(c->connect_timeout); + c->connect_timeout = NULL; } if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + /* Don't leak sockaddr if we're reconnecting */ + if (c->saddr) hi_free(c->saddr); + + sa = (struct sockaddr_un*)(c->saddr = hi_malloc(sizeof(struct sockaddr_un))); + if (sa == NULL) + goto oom; + c->addrlen = sizeof(struct sockaddr_un); sa->sun_family = AF_UNIX; strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); @@ -568,4 +606,7 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time errno = EPROTONOSUPPORT; return REDIS_ERR; #endif /* _WIN32 */ +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index a4393c06b..9f43283a5 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -38,8 +38,8 @@ #include "hiredis.h" void redisNetClose(redisContext *c); -int redisNetRead(redisContext *c, char *buf, size_t bufcap); -int redisNetWrite(redisContext *c); +ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap); +ssize_t redisNetWrite(redisContext *c); int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); @@ -51,4 +51,6 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time int redisKeepAlive(redisContext *c, int interval); int redisCheckConnectDone(redisContext *c, int *completed); +int redisSetTcpNoDelay(redisContext *c); + #endif diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index b9853ea9a..682b9a6b9 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -42,10 +42,14 @@ #include #include +#include "alloc.h" #include "read.h" #include "sds.h" #include "win32.h" +/* Initial size of our nested reply stack and how much we grow it when needd */ +#define REDIS_READER_STACK_SIZE 9 + static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -55,7 +59,7 @@ static void __redisReaderSetError(redisReader *r, int type, const char *str) { } /* Clear input buffer on errors. */ - sdsfree(r->buf); + hi_sdsfree(r->buf); r->buf = NULL; r->pos = r->len = 0; @@ -243,11 +247,12 @@ static void moveToNextTask(redisReader *r) { return; } - cur = &(r->rstack[r->ridx]); - prv = &(r->rstack[r->ridx-1]); + cur = r->task[r->ridx]; + prv = r->task[r->ridx-1]; assert(prv->type == REDIS_REPLY_ARRAY || prv->type == REDIS_REPLY_MAP || - prv->type == REDIS_REPLY_SET); + prv->type == REDIS_REPLY_SET || + prv->type == REDIS_REPLY_PUSH); if (cur->idx == prv->elements-1) { r->ridx--; } else { @@ -262,7 +267,7 @@ static void moveToNextTask(redisReader *r) { } static int processLineItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj; char *p; int len; @@ -297,7 +302,7 @@ static int processLineItem(redisReader *r) { if (strcasecmp(buf,",inf") == 0) { d = INFINITY; /* Positive infinite. */ } else if (strcasecmp(buf,",-inf") == 0) { - d = -INFINITY; /* Nevative infinite. */ + d = -INFINITY; /* Negative infinite. */ } else { d = strtod((char*)buf,&eptr); if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { @@ -344,7 +349,7 @@ static int processLineItem(redisReader *r) { } static int processBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj = NULL; char *p, *s; long long len; @@ -415,19 +420,43 @@ static int processBulkItem(redisReader *r) { return REDIS_ERR; } +static int redisReaderGrow(redisReader *r) { + redisReadTask **aux; + int newlen; + + /* Grow our stack size */ + newlen = r->tasks + REDIS_READER_STACK_SIZE; + aux = hi_realloc(r->task, sizeof(*r->task) * newlen); + if (aux == NULL) + goto oom; + + r->task = aux; + + /* Allocate new tasks */ + for (; r->tasks < newlen; r->tasks++) { + r->task[r->tasks] = hi_calloc(1, sizeof(**r->task)); + if (r->task[r->tasks] == NULL) + goto oom; + } + + return REDIS_OK; +oom: + __redisReaderSetErrorOOM(r); + return REDIS_ERR; +} + /* Process the array, map and set types. */ static int processAggregateItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj; char *p; long long elements; int root = 0, len; /* Set error for nested multi bulks with depth > 7 */ - if (r->ridx == 8) { - __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "No support for nested multi bulk replies with depth > 7"); - return REDIS_ERR; + if (r->ridx == r->tasks - 1) { + if (redisReaderGrow(r) == REDIS_ERR) + return REDIS_ERR; } if ((p = readLine(r,&len)) != NULL) { @@ -439,7 +468,9 @@ static int processAggregateItem(redisReader *r) { root = (r->ridx == 0); - if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX) || + (r->maxelements > 0 && elements > r->maxelements)) + { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "Multi-bulk length out of range"); return REDIS_ERR; @@ -475,12 +506,12 @@ static int processAggregateItem(redisReader *r) { cur->elements = elements; cur->obj = obj; r->ridx++; - r->rstack[r->ridx].type = -1; - r->rstack[r->ridx].elements = -1; - r->rstack[r->ridx].idx = 0; - r->rstack[r->ridx].obj = NULL; - r->rstack[r->ridx].parent = cur; - r->rstack[r->ridx].privdata = r->privdata; + r->task[r->ridx]->type = -1; + r->task[r->ridx]->elements = -1; + r->task[r->ridx]->idx = 0; + r->task[r->ridx]->obj = NULL; + r->task[r->ridx]->parent = cur; + r->task[r->ridx]->privdata = r->privdata; } else { moveToNextTask(r); } @@ -495,7 +526,7 @@ static int processAggregateItem(redisReader *r) { } static int processItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; char *p; /* check if we need to read type */ @@ -535,6 +566,9 @@ static int processItem(redisReader *r) { case '=': cur->type = REDIS_REPLY_VERB; break; + case '>': + cur->type = REDIS_REPLY_PUSH; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -560,6 +594,7 @@ static int processItem(redisReader *r) { case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: case REDIS_REPLY_SET: + case REDIS_REPLY_PUSH: return processAggregateItem(r); default: assert(NULL); @@ -570,33 +605,57 @@ static int processItem(redisReader *r) { redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { redisReader *r; - r = calloc(1,sizeof(redisReader)); + r = hi_calloc(1,sizeof(redisReader)); if (r == NULL) return NULL; - r->fn = fn; - r->buf = sdsempty(); - r->maxbuf = REDIS_READER_MAX_BUF; - if (r->buf == NULL) { - free(r); - return NULL; + r->buf = hi_sdsempty(); + if (r->buf == NULL) + goto oom; + + r->task = hi_calloc(REDIS_READER_STACK_SIZE, sizeof(*r->task)); + if (r->task == NULL) + goto oom; + + for (; r->tasks < REDIS_READER_STACK_SIZE; r->tasks++) { + r->task[r->tasks] = hi_calloc(1, sizeof(**r->task)); + if (r->task[r->tasks] == NULL) + goto oom; } + r->fn = fn; + r->maxbuf = REDIS_READER_MAX_BUF; + r->maxelements = REDIS_READER_MAX_ARRAY_ELEMENTS; r->ridx = -1; + return r; +oom: + redisReaderFree(r); + return NULL; } void redisReaderFree(redisReader *r) { if (r == NULL) return; + if (r->reply != NULL && r->fn && r->fn->freeObject) r->fn->freeObject(r->reply); - sdsfree(r->buf); - free(r); + + if (r->task) { + /* We know r->task[i] is allocated if i < r->tasks */ + for (int i = 0; i < r->tasks; i++) { + hi_free(r->task[i]); + } + + hi_free(r->task); + } + + hi_sdsfree(r->buf); + hi_free(r); } int redisReaderFeed(redisReader *r, const char *buf, size_t len) { - sds newbuf; + hisds newbuf; /* Return early when this reader is in an erroneous state. */ if (r->err) @@ -605,26 +664,25 @@ int redisReaderFeed(redisReader *r, const char *buf, size_t len) { /* Copy the provided buffer. */ if (buf != NULL && len >= 1) { /* Destroy internal buffer when it is empty and is quite large. */ - if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { - sdsfree(r->buf); - r->buf = sdsempty(); + if (r->len == 0 && r->maxbuf != 0 && hi_sdsavail(r->buf) > r->maxbuf) { + hi_sdsfree(r->buf); + r->buf = hi_sdsempty(); + if (r->buf == 0) goto oom; + r->pos = 0; - - /* r->buf should not be NULL since we just free'd a larger one. */ - assert(r->buf != NULL); } - newbuf = sdscatlen(r->buf,buf,len); - if (newbuf == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } + newbuf = hi_sdscatlen(r->buf,buf,len); + if (newbuf == NULL) goto oom; r->buf = newbuf; - r->len = sdslen(r->buf); + r->len = hi_sdslen(r->buf); } return REDIS_OK; +oom: + __redisReaderSetErrorOOM(r); + return REDIS_ERR; } int redisReaderGetReply(redisReader *r, void **reply) { @@ -642,12 +700,12 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Set first item to process when the stack is empty. */ if (r->ridx == -1) { - r->rstack[0].type = -1; - r->rstack[0].elements = -1; - r->rstack[0].idx = -1; - r->rstack[0].obj = NULL; - r->rstack[0].parent = NULL; - r->rstack[0].privdata = r->privdata; + r->task[0]->type = -1; + r->task[0]->elements = -1; + r->task[0]->idx = -1; + r->task[0]->obj = NULL; + r->task[0]->parent = NULL; + r->task[0]->privdata = r->privdata; r->ridx = 0; } @@ -663,9 +721,9 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Discard part of the buffer when we've consumed at least 1k, to avoid * doing unnecessary calls to memmove() in sds.c. */ if (r->pos >= 1024) { - sdsrange(r->buf,r->pos,-1); + if (hi_sdsrange(r->buf,r->pos,-1) < 0) return REDIS_ERR; r->pos = 0; - r->len = sdslen(r->buf); + r->len = hi_sdslen(r->buf); } /* Emit a reply when there is one. */ diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h index 58105312a..2d74d77a5 100644 --- a/deps/hiredis/read.h +++ b/deps/hiredis/read.h @@ -63,7 +63,11 @@ #define REDIS_REPLY_BIGNUM 13 #define REDIS_REPLY_VERB 14 -#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ +/* Default max unused reader buffer. */ +#define REDIS_READER_MAX_BUF (1024*16) + +/* Default multi-bulk element limit */ +#define REDIS_READER_MAX_ARRAY_ELEMENTS ((1LL<<32) - 1) #ifdef __cplusplus extern "C" { @@ -71,7 +75,7 @@ extern "C" { typedef struct redisReadTask { int type; - int elements; /* number of elements in multibulk container */ + long long elements; /* number of elements in multibulk container */ int idx; /* index in parent (array) object */ void *obj; /* holds user-generated value for a read task */ struct redisReadTask *parent; /* parent task */ @@ -96,8 +100,11 @@ typedef struct redisReader { size_t pos; /* Buffer cursor */ size_t len; /* Buffer length */ size_t maxbuf; /* Max length of unused buffer */ + long long maxelements; /* Max multi-bulk elements */ + + redisReadTask **task; + int tasks; - redisReadTask rstack[9]; int ridx; /* Index of current read task */ void *reply; /* Temporary reply pointer */ diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 6cf75841c..675e7649f 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -30,98 +30,100 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "fmacros.h" #include #include #include #include #include +#include #include "sds.h" #include "sdsalloc.h" -static inline int sdsHdrSize(char type) { - switch(type&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return sizeof(struct sdshdr5); - case SDS_TYPE_8: - return sizeof(struct sdshdr8); - case SDS_TYPE_16: - return sizeof(struct sdshdr16); - case SDS_TYPE_32: - return sizeof(struct sdshdr32); - case SDS_TYPE_64: - return sizeof(struct sdshdr64); +static inline int hi_sdsHdrSize(char type) { + switch(type&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: + return sizeof(struct hisdshdr5); + case HI_SDS_TYPE_8: + return sizeof(struct hisdshdr8); + case HI_SDS_TYPE_16: + return sizeof(struct hisdshdr16); + case HI_SDS_TYPE_32: + return sizeof(struct hisdshdr32); + case HI_SDS_TYPE_64: + return sizeof(struct hisdshdr64); } return 0; } -static inline char sdsReqType(size_t string_size) { +static inline char hi_sdsReqType(size_t string_size) { if (string_size < 32) - return SDS_TYPE_5; + return HI_SDS_TYPE_5; if (string_size < 0xff) - return SDS_TYPE_8; + return HI_SDS_TYPE_8; if (string_size < 0xffff) - return SDS_TYPE_16; + return HI_SDS_TYPE_16; if (string_size < 0xffffffff) - return SDS_TYPE_32; - return SDS_TYPE_64; + return HI_SDS_TYPE_32; + return HI_SDS_TYPE_64; } -/* Create a new sds string with the content specified by the 'init' pointer +/* Create a new hisds string with the content specified by the 'init' pointer * and 'initlen'. * If NULL is used for 'init' the string is initialized with zero bytes. * - * The string is always null-termined (all the sds strings are, always) so - * even if you create an sds string with: + * The string is always null-termined (all the hisds strings are, always) so + * even if you create an hisds string with: * - * mystring = sdsnewlen("abc",3); + * mystring = hi_sdsnewlen("abc",3); * * You can print the string with printf() as there is an implicit \0 at the * end of the string. However the string is binary safe and can contain - * \0 characters in the middle, as the length is stored in the sds header. */ -sds sdsnewlen(const void *init, size_t initlen) { + * \0 characters in the middle, as the length is stored in the hisds header. */ +hisds hi_sdsnewlen(const void *init, size_t initlen) { void *sh; - sds s; - char type = sdsReqType(initlen); + hisds s; + char type = hi_sdsReqType(initlen); /* Empty strings are usually created in order to append. Use type 8 * since type 5 is not good at this. */ - if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8; - int hdrlen = sdsHdrSize(type); + if (type == HI_SDS_TYPE_5 && initlen == 0) type = HI_SDS_TYPE_8; + int hdrlen = hi_sdsHdrSize(type); unsigned char *fp; /* flags pointer. */ - sh = s_malloc(hdrlen+initlen+1); + sh = hi_s_malloc(hdrlen+initlen+1); if (sh == NULL) return NULL; if (!init) memset(sh, 0, hdrlen+initlen+1); s = (char*)sh+hdrlen; fp = ((unsigned char*)s)-1; switch(type) { - case SDS_TYPE_5: { - *fp = type | (initlen << SDS_TYPE_BITS); + case HI_SDS_TYPE_5: { + *fp = type | (initlen << HI_SDS_TYPE_BITS); break; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); sh->len = initlen; sh->alloc = initlen; *fp = type; @@ -134,167 +136,164 @@ sds sdsnewlen(const void *init, size_t initlen) { return s; } -/* Create an empty (zero length) sds string. Even in this case the string +/* Create an empty (zero length) hisds string. Even in this case the string * always has an implicit null term. */ -sds sdsempty(void) { - return sdsnewlen("",0); +hisds hi_sdsempty(void) { + return hi_sdsnewlen("",0); } -/* Create a new sds string starting from a null terminated C string. */ -sds sdsnew(const char *init) { +/* Create a new hisds string starting from a null terminated C string. */ +hisds hi_sdsnew(const char *init) { size_t initlen = (init == NULL) ? 0 : strlen(init); - return sdsnewlen(init, initlen); + return hi_sdsnewlen(init, initlen); } -/* Duplicate an sds string. */ -sds sdsdup(const sds s) { - return sdsnewlen(s, sdslen(s)); +/* Duplicate an hisds string. */ +hisds hi_sdsdup(const hisds s) { + return hi_sdsnewlen(s, hi_sdslen(s)); } -/* Free an sds string. No operation is performed if 's' is NULL. */ -void sdsfree(sds s) { +/* Free an hisds string. No operation is performed if 's' is NULL. */ +void hi_sdsfree(hisds s) { if (s == NULL) return; - s_free((char*)s-sdsHdrSize(s[-1])); + hi_s_free((char*)s-hi_sdsHdrSize(s[-1])); } -/* Set the sds string length to the length as obtained with strlen(), so +/* Set the hisds string length to the length as obtained with strlen(), so * considering as content only up to the first null term character. * - * This function is useful when the sds string is hacked manually in some + * This function is useful when the hisds string is hacked manually in some * way, like in the following example: * - * s = sdsnew("foobar"); + * s = hi_sdsnew("foobar"); * s[2] = '\0'; - * sdsupdatelen(s); - * printf("%d\n", sdslen(s)); + * hi_sdsupdatelen(s); + * printf("%d\n", hi_sdslen(s)); * - * The output will be "2", but if we comment out the call to sdsupdatelen() + * The output will be "2", but if we comment out the call to hi_sdsupdatelen() * the output will be "6" as the string was modified but the logical length * remains 6 bytes. */ -void sdsupdatelen(sds s) { +void hi_sdsupdatelen(hisds s) { int reallen = strlen(s); - sdssetlen(s, reallen); + hi_sdssetlen(s, reallen); } -/* Modify an sds string in-place to make it empty (zero length). +/* Modify an hisds string in-place to make it empty (zero length). * However all the existing buffer is not discarded but set as free space * so that next append operations will not require allocations up to the * number of bytes previously available. */ -void sdsclear(sds s) { - sdssetlen(s, 0); +void hi_sdsclear(hisds s) { + hi_sdssetlen(s, 0); s[0] = '\0'; } -/* Enlarge the free space at the end of the sds string so that the caller +/* Enlarge the free space at the end of the hisds string so that the caller * is sure that after calling this function can overwrite up to addlen * bytes after the end of the string, plus one more byte for nul term. * - * Note: this does not change the *length* of the sds string as returned - * by sdslen(), but only the free buffer space we have. */ -sds sdsMakeRoomFor(sds s, size_t addlen) { + * Note: this does not change the *length* of the hisds string as returned + * by hi_sdslen(), but only the free buffer space we have. */ +hisds hi_sdsMakeRoomFor(hisds s, size_t addlen) { void *sh, *newsh; - size_t avail = sdsavail(s); + size_t avail = hi_sdsavail(s); size_t len, newlen; - char type, oldtype = s[-1] & SDS_TYPE_MASK; + char type, oldtype = s[-1] & HI_SDS_TYPE_MASK; int hdrlen; /* Return ASAP if there is enough space left. */ if (avail >= addlen) return s; - len = sdslen(s); - sh = (char*)s-sdsHdrSize(oldtype); + len = hi_sdslen(s); + sh = (char*)s-hi_sdsHdrSize(oldtype); newlen = (len+addlen); - if (newlen < SDS_MAX_PREALLOC) + if (newlen < HI_SDS_MAX_PREALLOC) newlen *= 2; else - newlen += SDS_MAX_PREALLOC; + newlen += HI_SDS_MAX_PREALLOC; - type = sdsReqType(newlen); + type = hi_sdsReqType(newlen); /* Don't use type 5: the user is appending to the string and type 5 is - * not able to remember empty space, so sdsMakeRoomFor() must be called + * not able to remember empty space, so hi_sdsMakeRoomFor() must be called * at every appending operation. */ - if (type == SDS_TYPE_5) type = SDS_TYPE_8; + if (type == HI_SDS_TYPE_5) type = HI_SDS_TYPE_8; - hdrlen = sdsHdrSize(type); + hdrlen = hi_sdsHdrSize(type); if (oldtype==type) { - newsh = s_realloc(sh, hdrlen+newlen+1); - if (newsh == NULL) { - s_free(sh); - return NULL; - } + newsh = hi_s_realloc(sh, hdrlen+newlen+1); + if (newsh == NULL) return NULL; s = (char*)newsh+hdrlen; } else { /* Since the header size changes, need to move the string forward, * and can't use realloc */ - newsh = s_malloc(hdrlen+newlen+1); + newsh = hi_s_malloc(hdrlen+newlen+1); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); - s_free(sh); + hi_s_free(sh); s = (char*)newsh+hdrlen; s[-1] = type; - sdssetlen(s, len); + hi_sdssetlen(s, len); } - sdssetalloc(s, newlen); + hi_sdssetalloc(s, newlen); return s; } -/* Reallocate the sds string so that it has no free space at the end. The +/* Reallocate the hisds string so that it has no free space at the end. The * contained string remains not altered, but next concatenation operations * will require a reallocation. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdsRemoveFreeSpace(sds s) { +hisds hi_sdsRemoveFreeSpace(hisds s) { void *sh, *newsh; - char type, oldtype = s[-1] & SDS_TYPE_MASK; + char type, oldtype = s[-1] & HI_SDS_TYPE_MASK; int hdrlen; - size_t len = sdslen(s); - sh = (char*)s-sdsHdrSize(oldtype); + size_t len = hi_sdslen(s); + sh = (char*)s-hi_sdsHdrSize(oldtype); - type = sdsReqType(len); - hdrlen = sdsHdrSize(type); + type = hi_sdsReqType(len); + hdrlen = hi_sdsHdrSize(type); if (oldtype==type) { - newsh = s_realloc(sh, hdrlen+len+1); + newsh = hi_s_realloc(sh, hdrlen+len+1); if (newsh == NULL) return NULL; s = (char*)newsh+hdrlen; } else { - newsh = s_malloc(hdrlen+len+1); + newsh = hi_s_malloc(hdrlen+len+1); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); - s_free(sh); + hi_s_free(sh); s = (char*)newsh+hdrlen; s[-1] = type; - sdssetlen(s, len); + hi_sdssetlen(s, len); } - sdssetalloc(s, len); + hi_sdssetalloc(s, len); return s; } -/* Return the total size of the allocation of the specifed sds string, +/* Return the total size of the allocation of the specifed hisds string, * including: - * 1) The sds header before the pointer. + * 1) The hisds header before the pointer. * 2) The string. * 3) The free buffer at the end if any. * 4) The implicit null term. */ -size_t sdsAllocSize(sds s) { - size_t alloc = sdsalloc(s); - return sdsHdrSize(s[-1])+alloc+1; +size_t hi_sdsAllocSize(hisds s) { + size_t alloc = hi_sdsalloc(s); + return hi_sdsHdrSize(s[-1])+alloc+1; } /* Return the pointer of the actual SDS allocation (normally SDS strings * are referenced by the start of the string buffer). */ -void *sdsAllocPtr(sds s) { - return (void*) (s-sdsHdrSize(s[-1])); +void *hi_sdsAllocPtr(hisds s) { + return (void*) (s-hi_sdsHdrSize(s[-1])); } -/* Increment the sds length and decrements the left free space at the +/* Increment the hisds length and decrements the left free space at the * end of the string according to 'incr'. Also set the null term * in the new end of the string. * * This function is used in order to fix the string length after the - * user calls sdsMakeRoomFor(), writes something after the end of + * user calls hi_sdsMakeRoomFor(), writes something after the end of * the current string, and finally needs to set the new length. * * Note: it is possible to use a negative increment in order to @@ -302,48 +301,48 @@ void *sdsAllocPtr(sds s) { * * Usage example: * - * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * Using hi_sdsIncrLen() and hi_sdsMakeRoomFor() it is possible to mount the * following schema, to cat bytes coming from the kernel to the end of an - * sds string without copying into an intermediate buffer: + * hisds string without copying into an intermediate buffer: * - * oldlen = sdslen(s); - * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * oldlen = hi_hi_sdslen(s); + * s = hi_sdsMakeRoomFor(s, BUFFER_SIZE); * nread = read(fd, s+oldlen, BUFFER_SIZE); * ... check for nread <= 0 and handle it ... - * sdsIncrLen(s, nread); + * hi_sdsIncrLen(s, nread); */ -void sdsIncrLen(sds s, int incr) { +void hi_sdsIncrLen(hisds s, int incr) { unsigned char flags = s[-1]; size_t len; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: { + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char oldlen = SDS_TYPE_5_LEN(flags); + unsigned char oldlen = HI_SDS_TYPE_5_LEN(flags); assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr))); - *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS); + *fp = HI_SDS_TYPE_5 | ((oldlen+incr) << HI_SDS_TYPE_BITS); len = oldlen+incr; break; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr))); len = (sh->len += incr); break; @@ -353,83 +352,83 @@ void sdsIncrLen(sds s, int incr) { s[len] = '\0'; } -/* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. +/* Grow the hisds to have the specified length. Bytes that were not part of + * the original length of the hisds will be set to zero. * * if the specified length is smaller than the current length, no operation * is performed. */ -sds sdsgrowzero(sds s, size_t len) { - size_t curlen = sdslen(s); +hisds hi_sdsgrowzero(hisds s, size_t len) { + size_t curlen = hi_sdslen(s); if (len <= curlen) return s; - s = sdsMakeRoomFor(s,len-curlen); + s = hi_sdsMakeRoomFor(s,len-curlen); if (s == NULL) return NULL; /* Make sure added region doesn't contain garbage */ memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ - sdssetlen(s, len); + hi_sdssetlen(s, len); return s; } /* Append the specified binary-safe string pointed by 't' of 'len' bytes to the - * end of the specified sds string 's'. + * end of the specified hisds string 's'. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatlen(sds s, const void *t, size_t len) { - size_t curlen = sdslen(s); +hisds hi_sdscatlen(hisds s, const void *t, size_t len) { + size_t curlen = hi_sdslen(s); - s = sdsMakeRoomFor(s,len); + s = hi_sdsMakeRoomFor(s,len); if (s == NULL) return NULL; memcpy(s+curlen, t, len); - sdssetlen(s, curlen+len); + hi_sdssetlen(s, curlen+len); s[curlen+len] = '\0'; return s; } -/* Append the specified null termianted C string to the sds string 's'. +/* Append the specified null termianted C string to the hisds string 's'. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscat(sds s, const char *t) { - return sdscatlen(s, t, strlen(t)); +hisds hi_sdscat(hisds s, const char *t) { + return hi_sdscatlen(s, t, strlen(t)); } -/* Append the specified sds 't' to the existing sds 's'. +/* Append the specified hisds 't' to the existing hisds 's'. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatsds(sds s, const sds t) { - return sdscatlen(s, t, sdslen(t)); +hisds hi_sdscatsds(hisds s, const hisds t) { + return hi_sdscatlen(s, t, hi_sdslen(t)); } -/* Destructively modify the sds string 's' to hold the specified binary +/* Destructively modify the hisds string 's' to hold the specified binary * safe string pointed by 't' of length 'len' bytes. */ -sds sdscpylen(sds s, const char *t, size_t len) { - if (sdsalloc(s) < len) { - s = sdsMakeRoomFor(s,len-sdslen(s)); +hisds hi_sdscpylen(hisds s, const char *t, size_t len) { + if (hi_sdsalloc(s) < len) { + s = hi_sdsMakeRoomFor(s,len-hi_sdslen(s)); if (s == NULL) return NULL; } memcpy(s, t, len); s[len] = '\0'; - sdssetlen(s, len); + hi_sdssetlen(s, len); return s; } -/* Like sdscpylen() but 't' must be a null-termined string so that the length +/* Like hi_sdscpylen() but 't' must be a null-termined string so that the length * of the string is obtained with strlen(). */ -sds sdscpy(sds s, const char *t) { - return sdscpylen(s, t, strlen(t)); +hisds hi_sdscpy(hisds s, const char *t) { + return hi_sdscpylen(s, t, strlen(t)); } -/* Helper for sdscatlonglong() doing the actual number -> string +/* Helper for hi_sdscatlonglong() doing the actual number -> string * conversion. 's' must point to a string with room for at least - * SDS_LLSTR_SIZE bytes. + * HI_SDS_LLSTR_SIZE bytes. * * The function returns the length of the null-terminated string * representation stored at 's'. */ -#define SDS_LLSTR_SIZE 21 -int sdsll2str(char *s, long long value) { +#define HI_SDS_LLSTR_SIZE 21 +int hi_sdsll2str(char *s, long long value) { char *p, aux; unsigned long long v; size_t l; @@ -460,8 +459,8 @@ int sdsll2str(char *s, long long value) { return l; } -/* Identical sdsll2str(), but for unsigned long long type. */ -int sdsull2str(char *s, unsigned long long v) { +/* Identical hi_sdsll2str(), but for unsigned long long type. */ +int hi_sdsull2str(char *s, unsigned long long v) { char *p, aux; size_t l; @@ -489,19 +488,19 @@ int sdsull2str(char *s, unsigned long long v) { return l; } -/* Create an sds string from a long long value. It is much faster than: +/* Create an hisds string from a long long value. It is much faster than: * - * sdscatprintf(sdsempty(),"%lld\n", value); + * hi_sdscatprintf(hi_sdsempty(),"%lld\n", value); */ -sds sdsfromlonglong(long long value) { - char buf[SDS_LLSTR_SIZE]; - int len = sdsll2str(buf,value); +hisds hi_sdsfromlonglong(long long value) { + char buf[HI_SDS_LLSTR_SIZE]; + int len = hi_sdsll2str(buf,value); - return sdsnewlen(buf,len); + return hi_sdsnewlen(buf,len); } -/* Like sdscatprintf() but gets va_list instead of being variadic. */ -sds sdscatvprintf(sds s, const char *fmt, va_list ap) { +/* Like hi_sdscatprintf() but gets va_list instead of being variadic. */ +hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap) { va_list cpy; char staticbuf[1024], *buf = staticbuf, *t; size_t buflen = strlen(fmt)*2; @@ -509,7 +508,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { /* We try to start using a static buffer for speed. * If not possible we revert to heap allocation. */ if (buflen > sizeof(staticbuf)) { - buf = s_malloc(buflen); + buf = hi_s_malloc(buflen); if (buf == NULL) return NULL; } else { buflen = sizeof(staticbuf); @@ -523,9 +522,9 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { vsnprintf(buf, buflen, fmt, cpy); va_end(cpy); if (buf[buflen-2] != '\0') { - if (buf != staticbuf) s_free(buf); + if (buf != staticbuf) hi_s_free(buf); buflen *= 2; - buf = s_malloc(buflen); + buf = hi_s_malloc(buflen); if (buf == NULL) return NULL; continue; } @@ -533,39 +532,39 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { } /* Finally concat the obtained string to the SDS string and return it. */ - t = sdscat(s, buf); - if (buf != staticbuf) s_free(buf); + t = hi_sdscat(s, buf); + if (buf != staticbuf) hi_s_free(buf); return t; } -/* Append to the sds string 's' a string obtained using printf-alike format +/* Append to the hisds string 's' a string obtained using printf-alike format * specifier. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. * * Example: * - * s = sdsnew("Sum is: "); - * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * s = hi_sdsnew("Sum is: "); + * s = hi_sdscatprintf(s,"%d+%d = %d",a,b,a+b). * * Often you need to create a string from scratch with the printf-alike - * format. When this is the need, just use sdsempty() as the target string: + * format. When this is the need, just use hi_sdsempty() as the target string: * - * s = sdscatprintf(sdsempty(), "... your format ...", args); + * s = hi_sdscatprintf(hi_sdsempty(), "... your format ...", args); */ -sds sdscatprintf(sds s, const char *fmt, ...) { +hisds hi_sdscatprintf(hisds s, const char *fmt, ...) { va_list ap; char *t; va_start(ap, fmt); - t = sdscatvprintf(s,fmt,ap); + t = hi_sdscatvprintf(s,fmt,ap); va_end(ap); return t; } -/* This function is similar to sdscatprintf, but much faster as it does +/* This function is similar to hi_sdscatprintf, but much faster as it does * not rely on sprintf() family functions implemented by the libc that - * are often very slow. Moreover directly handling the sds string as + * are often very slow. Moreover directly handling the hisds string as * new data is concatenated provides a performance improvement. * * However this function only handles an incompatible subset of printf-alike @@ -579,13 +578,13 @@ sds sdscatprintf(sds s, const char *fmt, ...) { * %U - 64 bit unsigned integer (unsigned long long, uint64_t) * %% - Verbatim "%" character. */ -sds sdscatfmt(sds s, char const *fmt, ...) { +hisds hi_sdscatfmt(hisds s, char const *fmt, ...) { const char *f = fmt; int i; va_list ap; va_start(ap,fmt); - i = sdslen(s); /* Position of the next byte to write to dest str. */ + i = hi_sdslen(s); /* Position of the next byte to write to dest str. */ while(*f) { char next, *str; size_t l; @@ -593,8 +592,8 @@ sds sdscatfmt(sds s, char const *fmt, ...) { unsigned long long unum; /* Make sure there is always space for at least 1 char. */ - if (sdsavail(s)==0) { - s = sdsMakeRoomFor(s,1); + if (hi_sdsavail(s)==0) { + s = hi_sdsMakeRoomFor(s,1); if (s == NULL) goto fmt_error; } @@ -606,13 +605,13 @@ sds sdscatfmt(sds s, char const *fmt, ...) { case 's': case 'S': str = va_arg(ap,char*); - l = (next == 's') ? strlen(str) : sdslen(str); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + l = (next == 's') ? strlen(str) : hi_sdslen(str); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); if (s == NULL) goto fmt_error; } memcpy(s+i,str,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; break; case 'i': @@ -622,14 +621,14 @@ sds sdscatfmt(sds s, char const *fmt, ...) { else num = va_arg(ap,long long); { - char buf[SDS_LLSTR_SIZE]; - l = sdsll2str(buf,num); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + char buf[HI_SDS_LLSTR_SIZE]; + l = hi_sdsll2str(buf,num); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; } break; @@ -640,26 +639,26 @@ sds sdscatfmt(sds s, char const *fmt, ...) { else unum = va_arg(ap,unsigned long long); { - char buf[SDS_LLSTR_SIZE]; - l = sdsull2str(buf,unum); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + char buf[HI_SDS_LLSTR_SIZE]; + l = hi_sdsull2str(buf,unum); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; } break; default: /* Handle %% and generally %. */ s[i++] = next; - sdsinclen(s,1); + hi_sdsinclen(s,1); break; } break; default: s[i++] = *f; - sdsinclen(s,1); + hi_sdsinclen(s,1); break; } f++; @@ -678,29 +677,29 @@ fmt_error: /* Remove the part of the string from left and from right composed just of * contiguous characters found in 'cset', that is a null terminted C string. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. * * Example: * - * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); - * s = sdstrim(s,"Aa. :"); + * s = hi_sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = hi_sdstrim(s,"Aa. :"); * printf("%s\n", s); * * Output will be just "Hello World". */ -sds sdstrim(sds s, const char *cset) { +hisds hi_sdstrim(hisds s, const char *cset) { char *start, *end, *sp, *ep; size_t len; sp = start = s; - ep = end = s+sdslen(s)-1; + ep = end = s+hi_sdslen(s)-1; while(sp <= end && strchr(cset, *sp)) sp++; while(ep > sp && strchr(cset, *ep)) ep--; len = (sp > ep) ? 0 : ((ep-sp)+1); if (s != sp) memmove(s, sp, len); s[len] = '\0'; - sdssetlen(s,len); + hi_sdssetlen(s,len); return s; } @@ -715,15 +714,20 @@ sds sdstrim(sds s, const char *cset) { * * The string is modified in-place. * + * Return value: + * -1 (error) if hi_sdslen(s) is larger than maximum positive ssize_t value. + * 0 on success. + * * Example: * - * s = sdsnew("Hello World"); - * sdsrange(s,1,-1); => "ello World" + * s = hi_sdsnew("Hello World"); + * hi_sdsrange(s,1,-1); => "ello World" */ -void sdsrange(sds s, int start, int end) { - size_t newlen, len = sdslen(s); +int hi_sdsrange(hisds s, ssize_t start, ssize_t end) { + size_t newlen, len = hi_sdslen(s); + if (len > SSIZE_MAX) return -1; - if (len == 0) return; + if (len == 0) return 0; if (start < 0) { start = len+start; if (start < 0) start = 0; @@ -734,9 +738,9 @@ void sdsrange(sds s, int start, int end) { } newlen = (start > end) ? 0 : (end-start)+1; if (newlen != 0) { - if (start >= (signed)len) { + if (start >= (ssize_t)len) { newlen = 0; - } else if (end >= (signed)len) { + } else if (end >= (ssize_t)len) { end = len-1; newlen = (start > end) ? 0 : (end-start)+1; } @@ -745,24 +749,25 @@ void sdsrange(sds s, int start, int end) { } if (start && newlen) memmove(s, s+start, newlen); s[newlen] = 0; - sdssetlen(s,newlen); + hi_sdssetlen(s,newlen); + return 0; } -/* Apply tolower() to every character of the sds string 's'. */ -void sdstolower(sds s) { - int len = sdslen(s), j; +/* Apply tolower() to every character of the hisds string 's'. */ +void hi_sdstolower(hisds s) { + int len = hi_sdslen(s), j; for (j = 0; j < len; j++) s[j] = tolower(s[j]); } -/* Apply toupper() to every character of the sds string 's'. */ -void sdstoupper(sds s) { - int len = sdslen(s), j; +/* Apply toupper() to every character of the hisds string 's'. */ +void hi_sdstoupper(hisds s) { + int len = hi_sdslen(s), j; for (j = 0; j < len; j++) s[j] = toupper(s[j]); } -/* Compare two sds strings s1 and s2 with memcmp(). +/* Compare two hisds strings s1 and s2 with memcmp(). * * Return value: * @@ -773,12 +778,12 @@ void sdstoupper(sds s) { * If two strings share exactly the same prefix, but one of the two has * additional characters, the longer string is considered to be greater than * the smaller one. */ -int sdscmp(const sds s1, const sds s2) { +int hi_sdscmp(const hisds s1, const hisds s2) { size_t l1, l2, minlen; int cmp; - l1 = sdslen(s1); - l2 = sdslen(s2); + l1 = hi_sdslen(s1); + l2 = hi_sdslen(s2); minlen = (l1 < l2) ? l1 : l2; cmp = memcmp(s1,s2,minlen); if (cmp == 0) return l1-l2; @@ -786,7 +791,7 @@ int sdscmp(const sds s1, const sds s2) { } /* Split 's' with separator in 'sep'. An array - * of sds strings is returned. *count will be set + * of hisds strings is returned. *count will be set * by reference to the number of tokens returned. * * On out of memory, zero length string, zero length @@ -794,20 +799,20 @@ int sdscmp(const sds s1, const sds s2) { * * Note that 'sep' is able to split a string using * a multi-character separator. For example - * sdssplit("foo_-_bar","_-_"); will return two + * hi_sdssplit("foo_-_bar","_-_"); will return two * elements "foo" and "bar". * * This version of the function is binary-safe but - * requires length arguments. sdssplit() is just the + * requires length arguments. hi_sdssplit() is just the * same function but for zero-terminated strings. */ -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { +hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { int elements = 0, slots = 5, start = 0, j; - sds *tokens; + hisds *tokens; if (seplen < 1 || len < 0) return NULL; - tokens = s_malloc(sizeof(sds)*slots); + tokens = hi_s_malloc(sizeof(hisds)*slots); if (tokens == NULL) return NULL; if (len == 0) { @@ -817,16 +822,16 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count for (j = 0; j < (len-(seplen-1)); j++) { /* make sure there is room for the next element and the final one */ if (slots < elements+2) { - sds *newtokens; + hisds *newtokens; slots *= 2; - newtokens = s_realloc(tokens,sizeof(sds)*slots); + newtokens = hi_s_realloc(tokens,sizeof(hisds)*slots); if (newtokens == NULL) goto cleanup; tokens = newtokens; } /* search the separator */ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { - tokens[elements] = sdsnewlen(s+start,j-start); + tokens[elements] = hi_sdsnewlen(s+start,j-start); if (tokens[elements] == NULL) goto cleanup; elements++; start = j+seplen; @@ -834,7 +839,7 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count } } /* Add the final element. We are sure there is room in the tokens array. */ - tokens[elements] = sdsnewlen(s+start,len-start); + tokens[elements] = hi_sdsnewlen(s+start,len-start); if (tokens[elements] == NULL) goto cleanup; elements++; *count = elements; @@ -843,62 +848,55 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count cleanup: { int i; - for (i = 0; i < elements; i++) sdsfree(tokens[i]); - s_free(tokens); + for (i = 0; i < elements; i++) hi_sdsfree(tokens[i]); + hi_s_free(tokens); *count = 0; return NULL; } } -/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ -void sdsfreesplitres(sds *tokens, int count) { +/* Free the result returned by hi_sdssplitlen(), or do nothing if 'tokens' is NULL. */ +void hi_sdsfreesplitres(hisds *tokens, int count) { if (!tokens) return; while(count--) - sdsfree(tokens[count]); - s_free(tokens); + hi_sdsfree(tokens[count]); + hi_s_free(tokens); } -/* Append to the sds string "s" an escaped string representation where +/* Append to the hisds string "s" an escaped string representation where * all the non-printable characters (tested with isprint()) are turned into * escapes in the form "\n\r\a...." or "\x". * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatrepr(sds s, const char *p, size_t len) { - s = sdscatlen(s,"\"",1); +hisds hi_sdscatrepr(hisds s, const char *p, size_t len) { + s = hi_sdscatlen(s,"\"",1); while(len--) { switch(*p) { case '\\': case '"': - s = sdscatprintf(s,"\\%c",*p); + s = hi_sdscatprintf(s,"\\%c",*p); break; - case '\n': s = sdscatlen(s,"\\n",2); break; - case '\r': s = sdscatlen(s,"\\r",2); break; - case '\t': s = sdscatlen(s,"\\t",2); break; - case '\a': s = sdscatlen(s,"\\a",2); break; - case '\b': s = sdscatlen(s,"\\b",2); break; + case '\n': s = hi_sdscatlen(s,"\\n",2); break; + case '\r': s = hi_sdscatlen(s,"\\r",2); break; + case '\t': s = hi_sdscatlen(s,"\\t",2); break; + case '\a': s = hi_sdscatlen(s,"\\a",2); break; + case '\b': s = hi_sdscatlen(s,"\\b",2); break; default: if (isprint(*p)) - s = sdscatprintf(s,"%c",*p); + s = hi_sdscatprintf(s,"%c",*p); else - s = sdscatprintf(s,"\\x%02x",(unsigned char)*p); + s = hi_sdscatprintf(s,"\\x%02x",(unsigned char)*p); break; } p++; } - return sdscatlen(s,"\"",1); + return hi_sdscatlen(s,"\"",1); } -/* Helper function for sdssplitargs() that returns non zero if 'c' - * is a valid hex digit. */ -int is_hex_digit(char c) { - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F'); -} - -/* Helper function for sdssplitargs() that converts a hex digit into an +/* Helper function for hi_sdssplitargs() that converts a hex digit into an * integer from 0 to 15 */ -int hex_digit_to_int(char c) { +static int hi_hex_digit_to_int(char c) { switch(c) { case '0': return 0; case '1': return 1; @@ -926,20 +924,20 @@ int hex_digit_to_int(char c) { * foo bar "newline are supported\n" and "\xff\x00otherstuff" * * The number of arguments is stored into *argc, and an array - * of sds is returned. + * of hisds is returned. * - * The caller should free the resulting array of sds strings with - * sdsfreesplitres(). + * The caller should free the resulting array of hisds strings with + * hi_sdsfreesplitres(). * - * Note that sdscatrepr() is able to convert back a string into - * a quoted string in the same format sdssplitargs() is able to parse. + * Note that hi_sdscatrepr() is able to convert back a string into + * a quoted string in the same format hi_sdssplitargs() is able to parse. * * The function returns the allocated tokens on success, even when the * input string is empty, or NULL if the input contains unbalanced * quotes or closed quotes followed by non space characters * as in: "foo"bar or "foo' */ -sds *sdssplitargs(const char *line, int *argc) { +hisds *hi_sdssplitargs(const char *line, int *argc) { const char *p = line; char *current = NULL; char **vector = NULL; @@ -954,18 +952,18 @@ sds *sdssplitargs(const char *line, int *argc) { int insq=0; /* set to 1 if we are in 'single quotes' */ int done=0; - if (current == NULL) current = sdsempty(); + if (current == NULL) current = hi_sdsempty(); while(!done) { if (inq) { if (*p == '\\' && *(p+1) == 'x' && - is_hex_digit(*(p+2)) && - is_hex_digit(*(p+3))) + isxdigit(*(p+2)) && + isxdigit(*(p+3))) { unsigned char byte; - byte = (hex_digit_to_int(*(p+2))*16)+ - hex_digit_to_int(*(p+3)); - current = sdscatlen(current,(char*)&byte,1); + byte = (hi_hex_digit_to_int(*(p+2))*16)+ + hi_hex_digit_to_int(*(p+3)); + current = hi_sdscatlen(current,(char*)&byte,1); p += 3; } else if (*p == '\\' && *(p+1)) { char c; @@ -979,7 +977,7 @@ sds *sdssplitargs(const char *line, int *argc) { case 'a': c = '\a'; break; default: c = *p; break; } - current = sdscatlen(current,&c,1); + current = hi_sdscatlen(current,&c,1); } else if (*p == '"') { /* closing quote must be followed by a space or * nothing at all. */ @@ -989,12 +987,12 @@ sds *sdssplitargs(const char *line, int *argc) { /* unterminated quotes */ goto err; } else { - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); } } else if (insq) { if (*p == '\\' && *(p+1) == '\'') { p++; - current = sdscatlen(current,"'",1); + current = hi_sdscatlen(current,"'",1); } else if (*p == '\'') { /* closing quote must be followed by a space or * nothing at all. */ @@ -1004,7 +1002,7 @@ sds *sdssplitargs(const char *line, int *argc) { /* unterminated quotes */ goto err; } else { - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); } } else { switch(*p) { @@ -1022,7 +1020,7 @@ sds *sdssplitargs(const char *line, int *argc) { insq=1; break; default: - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); break; } } @@ -1030,9 +1028,9 @@ sds *sdssplitargs(const char *line, int *argc) { } /* add the token to the vector */ { - char **new_vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); + char **new_vector = hi_s_realloc(vector,((*argc)+1)*sizeof(char*)); if (new_vector == NULL) { - s_free(vector); + hi_s_free(vector); return NULL; } @@ -1043,16 +1041,16 @@ sds *sdssplitargs(const char *line, int *argc) { } } else { /* Even on empty input string return something not NULL. */ - if (vector == NULL) vector = s_malloc(sizeof(void*)); + if (vector == NULL) vector = hi_s_malloc(sizeof(void*)); return vector; } } err: while((*argc)--) - sdsfree(vector[*argc]); - s_free(vector); - if (current) sdsfree(current); + hi_sdsfree(vector[*argc]); + hi_s_free(vector); + if (current) hi_sdsfree(current); *argc = 0; return NULL; } @@ -1061,13 +1059,13 @@ err: * characters specified in the 'from' string to the corresponding character * in the 'to' array. * - * For instance: sdsmapchars(mystring, "ho", "01", 2) + * For instance: hi_sdsmapchars(mystring, "ho", "01", 2) * will have the effect of turning the string "hello" into "0ell1". * - * The function returns the sds string pointer, that is always the same + * The function returns the hisds string pointer, that is always the same * as the input pointer since no resize is needed. */ -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { - size_t j, i, l = sdslen(s); +hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = hi_sdslen(s); for (j = 0; j < l; j++) { for (i = 0; i < setlen; i++) { @@ -1081,26 +1079,26 @@ sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { } /* Join an array of C strings using the specified separator (also a C string). - * Returns the result as an sds string. */ -sds sdsjoin(char **argv, int argc, char *sep) { - sds join = sdsempty(); + * Returns the result as an hisds string. */ +hisds hi_sdsjoin(char **argv, int argc, char *sep) { + hisds join = hi_sdsempty(); int j; for (j = 0; j < argc; j++) { - join = sdscat(join, argv[j]); - if (j != argc-1) join = sdscat(join,sep); + join = hi_sdscat(join, argv[j]); + if (j != argc-1) join = hi_sdscat(join,sep); } return join; } -/* Like sdsjoin, but joins an array of SDS strings. */ -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { - sds join = sdsempty(); +/* Like hi_sdsjoin, but joins an array of SDS strings. */ +hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen) { + hisds join = hi_sdsempty(); int j; for (j = 0; j < argc; j++) { - join = sdscatsds(join, argv[j]); - if (j != argc-1) join = sdscatlen(join,sep,seplen); + join = hi_sdscatsds(join, argv[j]); + if (j != argc-1) join = hi_sdscatlen(join,sep,seplen); } return join; } @@ -1110,138 +1108,138 @@ sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { * the overhead of function calls. Here we define these wrappers only for * the programs SDS is linked to, if they want to touch the SDS internals * even if they use a different allocator. */ -void *sds_malloc(size_t size) { return s_malloc(size); } -void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); } -void sds_free(void *ptr) { s_free(ptr); } +void *hi_sds_malloc(size_t size) { return hi_s_malloc(size); } +void *hi_sds_realloc(void *ptr, size_t size) { return hi_s_realloc(ptr,size); } +void hi_sds_free(void *ptr) { hi_s_free(ptr); } -#if defined(SDS_TEST_MAIN) +#if defined(HI_SDS_TEST_MAIN) #include #include "testhelp.h" #include "limits.h" #define UNUSED(x) (void)(x) -int sdsTest(void) { +int hi_sdsTest(void) { { - sds x = sdsnew("foo"), y; + hisds x = hi_sdsnew("foo"), y; test_cond("Create a string and obtain the length", - sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) + hi_sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) - sdsfree(x); - x = sdsnewlen("foo",2); + hi_sdsfree(x); + x = hi_sdsnewlen("foo",2); test_cond("Create a string with specified length", - sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) + hi_sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) - x = sdscat(x,"bar"); + x = hi_sdscat(x,"bar"); test_cond("Strings concatenation", - sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); + hi_sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); - x = sdscpy(x,"a"); - test_cond("sdscpy() against an originally longer string", - sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) + x = hi_sdscpy(x,"a"); + test_cond("hi_sdscpy() against an originally longer string", + hi_sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) - x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); - test_cond("sdscpy() against an originally shorter string", - sdslen(x) == 33 && + x = hi_sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); + test_cond("hi_sdscpy() against an originally shorter string", + hi_sdslen(x) == 33 && memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0) - sdsfree(x); - x = sdscatprintf(sdsempty(),"%d",123); - test_cond("sdscatprintf() seems working in the base case", - sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) + hi_sdsfree(x); + x = hi_sdscatprintf(hi_sdsempty(),"%d",123); + test_cond("hi_sdscatprintf() seems working in the base case", + hi_sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) - sdsfree(x); - x = sdsnew("--"); - x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); - test_cond("sdscatfmt() seems working in the base case", - sdslen(x) == 60 && + hi_sdsfree(x); + x = hi_sdsnew("--"); + x = hi_sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); + test_cond("hi_sdscatfmt() seems working in the base case", + hi_sdslen(x) == 60 && memcmp(x,"--Hello Hi! World -9223372036854775808," "9223372036854775807--",60) == 0) printf("[%s]\n",x); - sdsfree(x); - x = sdsnew("--"); - x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); - test_cond("sdscatfmt() seems working with unsigned numbers", - sdslen(x) == 35 && + hi_sdsfree(x); + x = hi_sdsnew("--"); + x = hi_sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); + test_cond("hi_sdscatfmt() seems working with unsigned numbers", + hi_sdslen(x) == 35 && memcmp(x,"--4294967295,18446744073709551615--",35) == 0) - sdsfree(x); - x = sdsnew(" x "); - sdstrim(x," x"); - test_cond("sdstrim() works when all chars match", - sdslen(x) == 0) + hi_sdsfree(x); + x = hi_sdsnew(" x "); + hi_sdstrim(x," x"); + test_cond("hi_sdstrim() works when all chars match", + hi_sdslen(x) == 0) - sdsfree(x); - x = sdsnew(" x "); - sdstrim(x," "); - test_cond("sdstrim() works when a single char remains", - sdslen(x) == 1 && x[0] == 'x') + hi_sdsfree(x); + x = hi_sdsnew(" x "); + hi_sdstrim(x," "); + test_cond("hi_sdstrim() works when a single char remains", + hi_sdslen(x) == 1 && x[0] == 'x') - sdsfree(x); - x = sdsnew("xxciaoyyy"); - sdstrim(x,"xy"); - test_cond("sdstrim() correctly trims characters", - sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) + hi_sdsfree(x); + x = hi_sdsnew("xxciaoyyy"); + hi_sdstrim(x,"xy"); + test_cond("hi_sdstrim() correctly trims characters", + hi_sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) - y = sdsdup(x); - sdsrange(y,1,1); - test_cond("sdsrange(...,1,1)", - sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) + y = hi_sdsdup(x); + hi_sdsrange(y,1,1); + test_cond("hi_sdsrange(...,1,1)", + hi_sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,-1); - test_cond("sdsrange(...,1,-1)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,1,-1); + test_cond("hi_sdsrange(...,1,-1)", + hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,-2,-1); - test_cond("sdsrange(...,-2,-1)", - sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,-2,-1); + test_cond("hi_sdsrange(...,-2,-1)", + hi_sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,2,1); - test_cond("sdsrange(...,2,1)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,2,1); + test_cond("hi_sdsrange(...,2,1)", + hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,100); - test_cond("sdsrange(...,1,100)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,1,100); + test_cond("hi_sdsrange(...,1,100)", + hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,100,100); - test_cond("sdsrange(...,100,100)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,100,100); + test_cond("hi_sdsrange(...,100,100)", + hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - sdsfree(y); - sdsfree(x); - x = sdsnew("foo"); - y = sdsnew("foa"); - test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0) + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("foo"); + y = hi_sdsnew("foa"); + test_cond("hi_sdscmp(foo,foa)", hi_sdscmp(x,y) > 0) - sdsfree(y); - sdsfree(x); - x = sdsnew("bar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0) + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("bar"); + y = hi_sdsnew("bar"); + test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) == 0) - sdsfree(y); - sdsfree(x); - x = sdsnew("aar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("aar"); + y = hi_sdsnew("bar"); + test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) < 0) - sdsfree(y); - sdsfree(x); - x = sdsnewlen("\a\n\0foo\r",7); - y = sdscatrepr(sdsempty(),x,sdslen(x)); - test_cond("sdscatrepr(...data...)", + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnewlen("\a\n\0foo\r",7); + y = hi_sdscatrepr(hi_sdsempty(),x,hi_sdslen(x)); + test_cond("hi_sdscatrepr(...data...)", memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) { @@ -1249,43 +1247,43 @@ int sdsTest(void) { char *p; int step = 10, j, i; - sdsfree(x); - sdsfree(y); - x = sdsnew("0"); - test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0); + hi_sdsfree(x); + hi_sdsfree(y); + x = hi_sdsnew("0"); + test_cond("hi_sdsnew() free/len buffers", hi_sdslen(x) == 1 && hi_sdsavail(x) == 0); /* Run the test a few times in order to hit the first two * SDS header types. */ for (i = 0; i < 10; i++) { - int oldlen = sdslen(x); - x = sdsMakeRoomFor(x,step); - int type = x[-1]&SDS_TYPE_MASK; + int oldlen = hi_sdslen(x); + x = hi_sdsMakeRoomFor(x,step); + int type = x[-1]&HI_SDS_TYPE_MASK; - test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen); - if (type != SDS_TYPE_5) { - test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step); - oldfree = sdsavail(x); + test_cond("sdsMakeRoomFor() len", hi_sdslen(x) == oldlen); + if (type != HI_SDS_TYPE_5) { + test_cond("hi_sdsMakeRoomFor() free", hi_sdsavail(x) >= step); + oldfree = hi_sdsavail(x); } p = x+oldlen; for (j = 0; j < step; j++) { p[j] = 'A'+j; } - sdsIncrLen(x,step); + hi_sdsIncrLen(x,step); } - test_cond("sdsMakeRoomFor() content", + test_cond("hi_sdsMakeRoomFor() content", memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0); - test_cond("sdsMakeRoomFor() final length",sdslen(x)==101); + test_cond("sdsMakeRoomFor() final length",hi_sdslen(x)==101); - sdsfree(x); + hi_sdsfree(x); } } - test_report() + test_report(); return 0; } #endif -#ifdef SDS_TEST_MAIN +#ifdef HI_SDS_TEST_MAIN int main(void) { - return sdsTest(); + return hi_sdsTest(); } #endif diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 0ce33b1fc..6aadf8ea2 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -30,29 +30,31 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef __SDS_H -#define __SDS_H +#ifndef HIREDIS_SDS_H +#define HIREDIS_SDS_H -#define SDS_MAX_PREALLOC (1024*1024) +#define HI_SDS_MAX_PREALLOC (1024*1024) #ifdef _MSC_VER #define __attribute__(x) +typedef long long ssize_t; +#define SSIZE_MAX (LLONG_MAX >> 1) #endif #include #include #include -typedef char *sds; +typedef char *hisds; /* Note: sdshdr5 is never used, we just access the flags byte directly. * However is here to document the layout of type 5 SDS strings. */ -struct __attribute__ ((__packed__)) sdshdr5 { +struct __attribute__ ((__packed__)) hisdshdr5 { unsigned char flags; /* 3 lsb of type, and 5 msb of string length */ #ifndef __cplusplus char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr8 { +struct __attribute__ ((__packed__)) hisdshdr8 { uint8_t len; /* used */ uint8_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -60,7 +62,7 @@ struct __attribute__ ((__packed__)) sdshdr8 { char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr16 { +struct __attribute__ ((__packed__)) hisdshdr16 { uint16_t len; /* used */ uint16_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -68,7 +70,7 @@ struct __attribute__ ((__packed__)) sdshdr16 { char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr32 { +struct __attribute__ ((__packed__)) hisdshdr32 { uint32_t len; /* used */ uint32_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -76,7 +78,7 @@ struct __attribute__ ((__packed__)) sdshdr32 { char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr64 { +struct __attribute__ ((__packed__)) hisdshdr64 { uint64_t len; /* used */ uint64_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -85,203 +87,203 @@ struct __attribute__ ((__packed__)) sdshdr64 { #endif }; -#define SDS_TYPE_5 0 -#define SDS_TYPE_8 1 -#define SDS_TYPE_16 2 -#define SDS_TYPE_32 3 -#define SDS_TYPE_64 4 -#define SDS_TYPE_MASK 7 -#define SDS_TYPE_BITS 3 -#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))); -#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)))) -#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS) +#define HI_SDS_TYPE_5 0 +#define HI_SDS_TYPE_8 1 +#define HI_SDS_TYPE_16 2 +#define HI_SDS_TYPE_32 3 +#define HI_SDS_TYPE_64 4 +#define HI_SDS_TYPE_MASK 7 +#define HI_SDS_TYPE_BITS 3 +#define HI_SDS_HDR_VAR(T,s) struct hisdshdr##T *sh = (struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T))); +#define HI_SDS_HDR(T,s) ((struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T)))) +#define HI_SDS_TYPE_5_LEN(f) ((f)>>HI_SDS_TYPE_BITS) -static inline size_t sdslen(const sds s) { +static inline size_t hi_sdslen(const hisds s) { unsigned char flags = s[-1]; - switch(__builtin_expect((flags&SDS_TYPE_MASK), SDS_TYPE_5)) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->len; - case SDS_TYPE_16: - return SDS_HDR(16,s)->len; - case SDS_TYPE_32: - return SDS_HDR(32,s)->len; - case SDS_TYPE_64: - return SDS_HDR(64,s)->len; + switch(__builtin_expect((flags&HI_SDS_TYPE_MASK), HI_SDS_TYPE_5)) { + case HI_SDS_TYPE_5: + return HI_SDS_TYPE_5_LEN(flags); + case HI_SDS_TYPE_8: + return HI_SDS_HDR(8,s)->len; + case HI_SDS_TYPE_16: + return HI_SDS_HDR(16,s)->len; + case HI_SDS_TYPE_32: + return HI_SDS_HDR(32,s)->len; + case HI_SDS_TYPE_64: + return HI_SDS_HDR(64,s)->len; } return 0; } -static inline size_t sdsavail(const sds s) { +static inline size_t hi_sdsavail(const hisds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: { + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { return 0; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); return sh->alloc - sh->len; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); return sh->alloc - sh->len; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); return sh->alloc - sh->len; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); return sh->alloc - sh->len; } } return 0; } -static inline void sdssetlen(sds s, size_t newlen) { +static inline void hi_sdssetlen(hisds s, size_t newlen) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); + *fp = (unsigned char)(HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS)); } break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len = (uint8_t)newlen; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->len = (uint8_t)newlen; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len = (uint16_t)newlen; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->len = (uint16_t)newlen; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len = (uint32_t)newlen; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->len = (uint32_t)newlen; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len = (uint64_t)newlen; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } -static inline void sdsinclen(sds s, size_t inc) { +static inline void hi_sdsinclen(hisds s, size_t inc) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + unsigned char newlen = HI_SDS_TYPE_5_LEN(flags)+(unsigned char)inc; + *fp = HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS); } break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len += (uint8_t)inc; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->len += (uint8_t)inc; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len += (uint16_t)inc; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->len += (uint16_t)inc; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len += (uint32_t)inc; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->len += (uint32_t)inc; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len += (uint64_t)inc; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->len += (uint64_t)inc; break; } } -/* sdsalloc() = sdsavail() + sdslen() */ -static inline size_t sdsalloc(const sds s) { +/* hi_sdsalloc() = hi_sdsavail() + hi_sdslen() */ +static inline size_t hi_sdsalloc(const hisds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->alloc; - case SDS_TYPE_16: - return SDS_HDR(16,s)->alloc; - case SDS_TYPE_32: - return SDS_HDR(32,s)->alloc; - case SDS_TYPE_64: - return SDS_HDR(64,s)->alloc; + switch(flags & HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: + return HI_SDS_TYPE_5_LEN(flags); + case HI_SDS_TYPE_8: + return HI_SDS_HDR(8,s)->alloc; + case HI_SDS_TYPE_16: + return HI_SDS_HDR(16,s)->alloc; + case HI_SDS_TYPE_32: + return HI_SDS_HDR(32,s)->alloc; + case HI_SDS_TYPE_64: + return HI_SDS_HDR(64,s)->alloc; } return 0; } -static inline void sdssetalloc(sds s, size_t newlen) { +static inline void hi_sdssetalloc(hisds s, size_t newlen) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: /* Nothing to do, this type has no total allocation info. */ break; - case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = (uint8_t)newlen; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = (uint16_t)newlen; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = (uint32_t)newlen; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = (uint64_t)newlen; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } -sds sdsnewlen(const void *init, size_t initlen); -sds sdsnew(const char *init); -sds sdsempty(void); -sds sdsdup(const sds s); -void sdsfree(sds s); -sds sdsgrowzero(sds s, size_t len); -sds sdscatlen(sds s, const void *t, size_t len); -sds sdscat(sds s, const char *t); -sds sdscatsds(sds s, const sds t); -sds sdscpylen(sds s, const char *t, size_t len); -sds sdscpy(sds s, const char *t); +hisds hi_sdsnewlen(const void *init, size_t initlen); +hisds hi_sdsnew(const char *init); +hisds hi_sdsempty(void); +hisds hi_sdsdup(const hisds s); +void hi_sdsfree(hisds s); +hisds hi_sdsgrowzero(hisds s, size_t len); +hisds hi_sdscatlen(hisds s, const void *t, size_t len); +hisds hi_sdscat(hisds s, const char *t); +hisds hi_sdscatsds(hisds s, const hisds t); +hisds hi_sdscpylen(hisds s, const char *t, size_t len); +hisds hi_sdscpy(hisds s, const char *t); -sds sdscatvprintf(sds s, const char *fmt, va_list ap); +hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap); #ifdef __GNUC__ -sds sdscatprintf(sds s, const char *fmt, ...) +hisds hi_sdscatprintf(hisds s, const char *fmt, ...) __attribute__((format(printf, 2, 3))); #else -sds sdscatprintf(sds s, const char *fmt, ...); +hisds hi_sdscatprintf(hisds s, const char *fmt, ...); #endif -sds sdscatfmt(sds s, char const *fmt, ...); -sds sdstrim(sds s, const char *cset); -void sdsrange(sds s, int start, int end); -void sdsupdatelen(sds s); -void sdsclear(sds s); -int sdscmp(const sds s1, const sds s2); -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); -void sdsfreesplitres(sds *tokens, int count); -void sdstolower(sds s); -void sdstoupper(sds s); -sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, const char *p, size_t len); -sds *sdssplitargs(const char *line, int *argc); -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); -sds sdsjoin(char **argv, int argc, char *sep); -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); +hisds hi_sdscatfmt(hisds s, char const *fmt, ...); +hisds hi_sdstrim(hisds s, const char *cset); +int hi_sdsrange(hisds s, ssize_t start, ssize_t end); +void hi_sdsupdatelen(hisds s); +void hi_sdsclear(hisds s); +int hi_sdscmp(const hisds s1, const hisds s2); +hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); +void hi_sdsfreesplitres(hisds *tokens, int count); +void hi_sdstolower(hisds s); +void hi_sdstoupper(hisds s); +hisds hi_sdsfromlonglong(long long value); +hisds hi_sdscatrepr(hisds s, const char *p, size_t len); +hisds *hi_sdssplitargs(const char *line, int *argc); +hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen); +hisds hi_sdsjoin(char **argv, int argc, char *sep); +hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen); /* Low level functions exposed to the user API */ -sds sdsMakeRoomFor(sds s, size_t addlen); -void sdsIncrLen(sds s, int incr); -sds sdsRemoveFreeSpace(sds s); -size_t sdsAllocSize(sds s); -void *sdsAllocPtr(sds s); +hisds hi_sdsMakeRoomFor(hisds s, size_t addlen); +void hi_sdsIncrLen(hisds s, int incr); +hisds hi_sdsRemoveFreeSpace(hisds s); +size_t hi_sdsAllocSize(hisds s); +void *hi_sdsAllocPtr(hisds s); /* Export the allocator used by SDS to the program using SDS. * Sometimes the program SDS is linked to, may use a different set of * allocators, but may want to allocate or free things that SDS will * respectively free or allocate. */ -void *sds_malloc(size_t size); -void *sds_realloc(void *ptr, size_t size); -void sds_free(void *ptr); +void *hi_sds_malloc(size_t size); +void *hi_sds_realloc(void *ptr, size_t size); +void hi_sds_free(void *ptr); #ifdef REDIS_TEST -int sdsTest(int argc, char *argv[]); +int hi_sdsTest(int argc, char *argv[]); #endif -#endif +#endif /* HIREDIS_SDS_H */ diff --git a/deps/hiredis/sdsalloc.h b/deps/hiredis/sdsalloc.h index f43023c48..c9dcc3df8 100644 --- a/deps/hiredis/sdsalloc.h +++ b/deps/hiredis/sdsalloc.h @@ -37,6 +37,8 @@ * the include of your alternate allocator if needed (not needed in order * to use the default libc allocator). */ -#define s_malloc malloc -#define s_realloc realloc -#define s_free free +#include "alloc.h" + +#define hi_s_malloc hi_malloc +#define hi_s_realloc hi_realloc +#define hi_s_free hi_free diff --git a/deps/hiredis/sdscompat.h b/deps/hiredis/sdscompat.h new file mode 100644 index 000000000..e5a2574f3 --- /dev/null +++ b/deps/hiredis/sdscompat.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * SDS compatibility header. + * + * This simple file maps sds types and calls to their unique hiredis symbol names. + * It's useful when we build Hiredis as a dependency of Redis and want to call + * Hiredis' sds symbols rather than the ones built into Redis, as the libraries + * have slightly diverged and could cause hard to track down ABI incompatibility + * bugs. + * + */ + +#ifndef HIREDIS_SDS_COMPAT +#define HIREDIS_SDS_COMPAT + +#define sds hisds + +#define sdslen hi_sdslen +#define sdsavail hi_sdsavail +#define sdssetlen hi_sdssetlen +#define sdsinclen hi_sdsinclen +#define sdsalloc hi_sdsalloc +#define sdssetalloc hi_sdssetalloc + +#define sdsAllocPtr hi_sdsAllocPtr +#define sdsAllocSize hi_sdsAllocSize +#define sdscat hi_sdscat +#define sdscatfmt hi_sdscatfmt +#define sdscatlen hi_sdscatlen +#define sdscatprintf hi_sdscatprintf +#define sdscatrepr hi_sdscatrepr +#define sdscatsds hi_sdscatsds +#define sdscatvprintf hi_sdscatvprintf +#define sdsclear hi_sdsclear +#define sdscmp hi_sdscmp +#define sdscpy hi_sdscpy +#define sdscpylen hi_sdscpylen +#define sdsdup hi_sdsdup +#define sdsempty hi_sdsempty +#define sds_free hi_sds_free +#define sdsfree hi_sdsfree +#define sdsfreesplitres hi_sdsfreesplitres +#define sdsfromlonglong hi_sdsfromlonglong +#define sdsgrowzero hi_sdsgrowzero +#define sdsIncrLen hi_sdsIncrLen +#define sdsjoin hi_sdsjoin +#define sdsjoinsds hi_sdsjoinsds +#define sdsll2str hi_sdsll2str +#define sdsMakeRoomFor hi_sdsMakeRoomFor +#define sds_malloc hi_sds_malloc +#define sdsmapchars hi_sdsmapchars +#define sdsnew hi_sdsnew +#define sdsnewlen hi_sdsnewlen +#define sdsrange hi_sdsrange +#define sds_realloc hi_sds_realloc +#define sdsRemoveFreeSpace hi_sdsRemoveFreeSpace +#define sdssplitargs hi_sdssplitargs +#define sdssplitlen hi_sdssplitlen +#define sdstolower hi_sdstolower +#define sdstoupper hi_sdstoupper +#define sdstrim hi_sdstrim +#define sdsull2str hi_sdsull2str +#define sdsupdatelen hi_sdsupdatelen + +#endif /* HIREDIS_SDS_COMPAT */ diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c index 4cc2f414f..f99d14b05 100644 --- a/deps/hiredis/sockcompat.c +++ b/deps/hiredis/sockcompat.c @@ -212,7 +212,7 @@ int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, sockle int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { int ret = 0; if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { - struct timeval *tv = optval; + const struct timeval *tv = optval; DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); } else { diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h index 56006c163..85810e848 100644 --- a/deps/hiredis/sockcompat.h +++ b/deps/hiredis/sockcompat.h @@ -49,9 +49,10 @@ #include #include #include +#include #ifdef _MSC_VER -typedef signed long ssize_t; +typedef long long ssize_t; #endif /* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c index 78ab9e43e..fe9a2fdce 100644 --- a/deps/hiredis/ssl.c +++ b/deps/hiredis/ssl.c @@ -34,25 +34,33 @@ #include "async.h" #include -#include #include #include +#ifdef _WIN32 +#include +#else +#include +#endif #include #include +#include "win32.h" #include "async_private.h" +#include "hiredis_ssl.h" void __redisSetError(redisContext *c, int type, const char *str); -/* The SSL context is attached to SSL/TLS connections as a privdata. */ -typedef struct redisSSLContext { - /** - * OpenSSL SSL_CTX; It is optional and will not be set when using - * user-supplied SSL. - */ +struct redisSSLContext { + /* Associated OpenSSL SSL_CTX as created by redisCreateSSLContext() */ SSL_CTX *ssl_ctx; + /* Requested SNI, or NULL */ + char *server_name; +}; + +/* The SSL connection context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSL { /** * OpenSSL SSL object. */ @@ -72,43 +80,11 @@ typedef struct redisSSLContext { * should resume whenever a read takes place, if possible */ int pendingWrite; -} redisSSLContext; +} redisSSL; /* Forward declaration */ redisContextFuncs redisContextSSLFuncs; -#ifdef HIREDIS_SSL_TRACE -/** - * Callback used for debugging - */ -static void sslLogCallback(const SSL *ssl, int where, int ret) { - const char *retstr = ""; - int should_log = 1; - /* Ignore low-level SSL stuff */ - - if (where & SSL_CB_ALERT) { - should_log = 1; - } - if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { - should_log = 1; - } - if ((where & SSL_CB_EXIT) && ret == 0) { - should_log = 1; - } - - if (!should_log) { - return; - } - - retstr = SSL_alert_type_string(ret); - printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); - - if (where == SSL_CB_HANDSHAKE_DONE) { - printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); - } -} -#endif - /** * OpenSSL global initialization and locking handling callbacks. * Note that this is only required for OpenSSL < 1.1.0. @@ -119,6 +95,18 @@ static void sslLogCallback(const SSL *ssl, int where, int ret) { #endif #ifdef HIREDIS_USE_CRYPTO_LOCKS +#ifdef _WIN32 +typedef CRITICAL_SECTION sslLockType; +static void sslLockInit(sslLockType* l) { + InitializeCriticalSection(l); +} +static void sslLockAcquire(sslLockType* l) { + EnterCriticalSection(l); +} +static void sslLockRelease(sslLockType* l) { + LeaveCriticalSection(l); +} +#else typedef pthread_mutex_t sslLockType; static void sslLockInit(sslLockType *l) { pthread_mutex_init(l, NULL); @@ -129,7 +117,9 @@ static void sslLockAcquire(sslLockType *l) { static void sslLockRelease(sslLockType *l) { pthread_mutex_unlock(l); } -static pthread_mutex_t *ossl_locks; +#endif + +static sslLockType* ossl_locks; static void opensslDoLock(int mode, int lkid, const char *f, int line) { sslLockType *l = ossl_locks + lkid; @@ -144,36 +134,151 @@ static void opensslDoLock(int mode, int lkid, const char *f, int line) { (void)line; } -static void initOpensslLocks(void) { +static int initOpensslLocks(void) { unsigned ii, nlocks; if (CRYPTO_get_locking_callback() != NULL) { /* Someone already set the callback before us. Don't destroy it! */ - return; + return REDIS_OK; } nlocks = CRYPTO_num_locks(); - ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + ossl_locks = hi_malloc(sizeof(*ossl_locks) * nlocks); + if (ossl_locks == NULL) + return REDIS_ERR; + for (ii = 0; ii < nlocks; ii++) { sslLockInit(ossl_locks + ii); } CRYPTO_set_locking_callback(opensslDoLock); + return REDIS_OK; } #endif /* HIREDIS_USE_CRYPTO_LOCKS */ +int redisInitOpenSSL(void) +{ + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + + return REDIS_OK; +} + +/** + * redisSSLContext helper context destruction. + */ + +const char *redisSSLContextGetError(redisSSLContextError error) +{ + switch (error) { + case REDIS_SSL_CTX_NONE: + return "No Error"; + case REDIS_SSL_CTX_CREATE_FAILED: + return "Failed to create OpenSSL SSL_CTX"; + case REDIS_SSL_CTX_CERT_KEY_REQUIRED: + return "Client cert and key must both be specified or skipped"; + case REDIS_SSL_CTX_CA_CERT_LOAD_FAILED: + return "Failed to load CA Certificate or CA Path"; + case REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED: + return "Failed to load client certificate"; + case REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED: + return "Failed to load private key"; + default: + return "Unknown error code"; + } +} + +void redisFreeSSLContext(redisSSLContext *ctx) +{ + if (!ctx) + return; + + if (ctx->server_name) { + hi_free(ctx->server_name); + ctx->server_name = NULL; + } + + if (ctx->ssl_ctx) { + SSL_CTX_free(ctx->ssl_ctx); + ctx->ssl_ctx = NULL; + } + + hi_free(ctx); +} + + +/** + * redisSSLContext helper context initialization. + */ + +redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath, + const char *cert_filename, const char *private_key_filename, + const char *server_name, redisSSLContextError *error) +{ + redisSSLContext *ctx = hi_calloc(1, sizeof(redisSSLContext)); + if (ctx == NULL) + goto error; + + ctx->ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ctx->ssl_ctx) { + if (error) *error = REDIS_SSL_CTX_CREATE_FAILED; + goto error; + } + + SSL_CTX_set_options(ctx->ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ctx->ssl_ctx, SSL_VERIFY_PEER, NULL); + + if ((cert_filename != NULL && private_key_filename == NULL) || + (private_key_filename != NULL && cert_filename == NULL)) { + if (error) *error = REDIS_SSL_CTX_CERT_KEY_REQUIRED; + goto error; + } + + if (capath || cacert_filename) { + if (!SSL_CTX_load_verify_locations(ctx->ssl_ctx, cacert_filename, capath)) { + if (error) *error = REDIS_SSL_CTX_CA_CERT_LOAD_FAILED; + goto error; + } + } + + if (cert_filename) { + if (!SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, cert_filename)) { + if (error) *error = REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED; + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, private_key_filename, SSL_FILETYPE_PEM)) { + if (error) *error = REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED; + goto error; + } + } + + if (server_name) + ctx->server_name = hi_strdup(server_name); + + return ctx; + +error: + redisFreeSSLContext(ctx); + return NULL; +} + /** * SSL Connection initialization. */ -static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { - if (c->privdata) { + +static int redisSSLConnect(redisContext *c, SSL *ssl) { + if (c->privctx) { __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); return REDIS_ERR; } - c->privdata = calloc(1, sizeof(redisSSLContext)); + + redisSSL *rssl = hi_calloc(1, sizeof(redisSSL)); + if (rssl == NULL) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; + } c->funcs = &redisContextSSLFuncs; - redisSSLContext *rssl = c->privdata; - - rssl->ssl_ctx = ssl_ctx; rssl->ssl = ssl; SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); @@ -183,12 +288,14 @@ static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { ERR_clear_error(); int rv = SSL_connect(rssl->ssl); if (rv == 1) { + c->privctx = rssl; return REDIS_OK; } rv = SSL_get_error(rssl->ssl, rv); if (((c->flags & REDIS_BLOCK) == 0) && (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + c->privctx = rssl; return REDIS_OK; } @@ -203,83 +310,58 @@ static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { } __redisSetError(c, REDIS_ERR_IO, err); } + + hi_free(rssl); return REDIS_ERR; } +/** + * A wrapper around redisSSLConnect() for users who manage their own context and + * create their own SSL object. + */ + int redisInitiateSSL(redisContext *c, SSL *ssl) { - return redisSSLConnect(c, NULL, ssl); + return redisSSLConnect(c, ssl); } -int redisSecureConnection(redisContext *c, const char *capath, - const char *certpath, const char *keypath, const char *servername) { +/** + * A wrapper around redisSSLConnect() for users who use redisSSLContext and don't + * manage their own SSL objects. + */ - SSL_CTX *ssl_ctx = NULL; - SSL *ssl = NULL; +int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx) +{ + if (!c || !redis_ssl_ctx) + return REDIS_ERR; - /* Initialize global OpenSSL stuff */ - static int isInit = 0; - if (!isInit) { - isInit = 1; - SSL_library_init(); -#ifdef HIREDIS_USE_CRYPTO_LOCKS - initOpensslLocks(); -#endif - } + /* We want to verify that redisSSLConnect() won't fail on this, as it will + * not own the SSL object in that case and we'll end up leaking. + */ + if (c->privctx) + return REDIS_ERR; - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); - if (!ssl_ctx) { - __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); - goto error; - } - -#ifdef HIREDIS_SSL_TRACE - SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); -#endif - SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); - SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); - if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { - __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); - goto error; - } - - if (capath) { - if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { - __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); - goto error; - } - } - if (certpath) { - if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { - __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); - goto error; - } - if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { - __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); - goto error; - } - } - - ssl = SSL_new(ssl_ctx); + SSL *ssl = SSL_new(redis_ssl_ctx->ssl_ctx); if (!ssl) { __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); goto error; } - if (servername) { - if (!SSL_set_tlsext_host_name(ssl, servername)) { - __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + + if (redis_ssl_ctx->server_name) { + if (!SSL_set_tlsext_host_name(ssl, redis_ssl_ctx->server_name)) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to set server_name/SNI"); goto error; } } - return redisSSLConnect(c, ssl_ctx, ssl); + return redisSSLConnect(c, ssl); error: - if (ssl) SSL_free(ssl); - if (ssl_ctx) SSL_CTX_free(ssl_ctx); + if (ssl) + SSL_free(ssl); return REDIS_ERR; } -static int maybeCheckWant(redisSSLContext *rssl, int rv) { +static int maybeCheckWant(redisSSL *rssl, int rv) { /** * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set * and true is returned. False is returned otherwise @@ -299,23 +381,19 @@ static int maybeCheckWant(redisSSLContext *rssl, int rv) { * Implementation of redisContextFuncs for SSL connections. */ -static void redisSSLFreeContext(void *privdata){ - redisSSLContext *rsc = privdata; +static void redisSSLFree(void *privctx){ + redisSSL *rsc = privctx; if (!rsc) return; if (rsc->ssl) { SSL_free(rsc->ssl); rsc->ssl = NULL; } - if (rsc->ssl_ctx) { - SSL_CTX_free(rsc->ssl_ctx); - rsc->ssl_ctx = NULL; - } - free(rsc); + hi_free(rsc); } -static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { - redisSSLContext *rssl = c->privdata; +static ssize_t redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSL *rssl = c->privctx; int nread = SSL_read(rssl->ssl, buf, bufcap); if (nread > 0) { @@ -356,10 +434,10 @@ static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { } } -static int redisSSLWrite(redisContext *c) { - redisSSLContext *rssl = c->privdata; +static ssize_t redisSSLWrite(redisContext *c) { + redisSSL *rssl = c->privctx; - size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + size_t len = rssl->lastLen ? rssl->lastLen : hi_sdslen(c->obuf); int rv = SSL_write(rssl->ssl, c->obuf, len); if (rv > 0) { @@ -380,7 +458,7 @@ static int redisSSLWrite(redisContext *c) { static void redisSSLAsyncRead(redisAsyncContext *ac) { int rv; - redisSSLContext *rssl = ac->c.privdata; + redisSSL *rssl = ac->c.privctx; redisContext *c = &ac->c; rssl->wantRead = 0; @@ -410,7 +488,7 @@ static void redisSSLAsyncRead(redisAsyncContext *ac) { static void redisSSLAsyncWrite(redisAsyncContext *ac) { int rv, done = 0; - redisSSLContext *rssl = ac->c.privdata; + redisSSL *rssl = ac->c.privctx; redisContext *c = &ac->c; rssl->pendingWrite = 0; @@ -439,7 +517,7 @@ static void redisSSLAsyncWrite(redisAsyncContext *ac) { } redisContextFuncs redisContextSSLFuncs = { - .free_privdata = redisSSLFreeContext, + .free_privctx = redisSSLFree, .async_read = redisSSLAsyncRead, .async_write = redisSSLAsyncWrite, .read = redisSSLRead, diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 8668e1856..829536739 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -1,22 +1,24 @@ #include "fmacros.h" +#include "sockcompat.h" #include #include #include +#ifndef _WIN32 #include -#include #include -#include +#endif #include -#include #include #include #include #include "hiredis.h" +#include "async.h" #ifdef HIREDIS_TEST_SSL #include "hiredis_ssl.h" #endif #include "net.h" +#include "win32.h" enum connection_type { CONN_TCP, @@ -47,15 +49,35 @@ struct config { } ssl; }; +struct privdata { + int dtor_counter; +}; + +struct pushCounters { + int nil; + int str; +}; + +#ifdef HIREDIS_TEST_SSL +redisSSLContext *_ssl_ctx = NULL; +#endif + /* The following lines make up our testing "framework" :) */ -static int tests = 0, fails = 0; +static int tests = 0, fails = 0, skips = 0; #define test(_s) { printf("#%02d ", ++tests); printf(_s); } #define test_cond(_c) if(_c) printf("\033[0;32mPASSED\033[0;0m\n"); else {printf("\033[0;31mFAILED\033[0;0m\n"); fails++;} +#define test_skipped() { printf("\033[01;33mSKIPPED\033[0;0m\n"); skips++; } static long long usec(void) { +#ifndef _MSC_VER struct timeval tv; gettimeofday(&tv,NULL); return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; +#else + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + return (((long long)ft.dwHighDateTime << 32) | ft.dwLowDateTime) / 10; +#endif } /* The assert() calls below have side effects, so we need assert() @@ -65,6 +87,43 @@ static long long usec(void) { #define assert(e) (void)(e) #endif +/* Helper to extract Redis version information. Aborts on any failure. */ +#define REDIS_VERSION_FIELD "redis_version:" +void get_redis_version(redisContext *c, int *majorptr, int *minorptr) { + redisReply *reply; + char *eptr, *s, *e; + int major, minor; + + reply = redisCommand(c, "INFO"); + if (reply == NULL || c->err || reply->type != REDIS_REPLY_STRING) + goto abort; + if ((s = strstr(reply->str, REDIS_VERSION_FIELD)) == NULL) + goto abort; + + s += strlen(REDIS_VERSION_FIELD); + + /* We need a field terminator and at least 'x.y.z' (5) bytes of data */ + if ((e = strstr(s, "\r\n")) == NULL || (e - s) < 5) + goto abort; + + /* Extract version info */ + major = strtol(s, &eptr, 10); + if (*eptr != '.') goto abort; + minor = strtol(eptr+1, NULL, 10); + + /* Push info the caller wants */ + if (majorptr) *majorptr = major; + if (minorptr) *minorptr = minor; + + freeReplyObject(reply); + return; + +abort: + freeReplyObject(reply); + fprintf(stderr, "Error: Cannot determine Redis version, aborting\n"); + exit(1); +} + static redisContext *select_database(redisContext *c) { redisReply *reply; @@ -87,6 +146,26 @@ static redisContext *select_database(redisContext *c) { return c; } +/* Switch protocol */ +static void send_hello(redisContext *c, int version) { + redisReply *reply; + int expected; + + reply = redisCommand(c, "HELLO %d", version); + expected = version == 3 ? REDIS_REPLY_MAP : REDIS_REPLY_ARRAY; + assert(reply != NULL && reply->type == expected); + freeReplyObject(reply); +} + +/* Togggle client tracking */ +static void send_client_tracking(redisContext *c, const char *str) { + redisReply *reply; + + reply = redisCommand(c, "CLIENT TRACKING %s", str); + assert(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); +} + static int disconnect(redisContext *c, int keep_fd) { redisReply *reply; @@ -105,9 +184,9 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } -static void do_ssl_handshake(redisContext *c, struct config config) { +static void do_ssl_handshake(redisContext *c) { #ifdef HIREDIS_TEST_SSL - redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + redisInitiateSSLWithContext(c, _ssl_ctx); if (c->err) { printf("SSL error: %s\n", c->errstr); redisFree(c); @@ -115,7 +194,6 @@ static void do_ssl_handshake(redisContext *c, struct config config) { } #else (void) c; - (void) config; #endif } @@ -150,7 +228,7 @@ static redisContext *do_connect(struct config config) { } if (config.type == CONN_SSL) { - do_ssl_handshake(c, config); + do_ssl_handshake(c); } return select_database(c); @@ -160,7 +238,7 @@ static void do_reconnect(redisContext *c, struct config config) { redisReconnect(c); if (config.type == CONN_SSL) { - do_ssl_handshake(c, config); + do_ssl_handshake(c); } } @@ -172,43 +250,43 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"SET foo bar"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%s string interpolation: "); len = redisFormatCommand(&cmd,"SET %s %s","foo","bar"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%s and an empty string: "); len = redisFormatCommand(&cmd,"SET %s %s","foo",""); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); + hi_free(cmd); test("Format command with an empty string in between proper interpolations: "); len = redisFormatCommand(&cmd,"SET %s %s","","foo"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$0\r\n\r\n$3\r\nfoo\r\n",len) == 0 && len == 4+4+(3+2)+4+(0+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%b string interpolation: "); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%b and an empty string: "); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); + hi_free(cmd); test("Format command with literal %%: "); len = redisFormatCommand(&cmd,"SET %% %%"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$1\r\n%\r\n$1\r\n%\r\n",len) == 0 && len == 4+4+(3+2)+4+(1+2)+4+(1+2)); - free(cmd); + hi_free(cmd); /* Vararg width depends on the type. These tests make sure that the * width is correctly determined using the format and subsequent varargs @@ -219,7 +297,7 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"key:%08" fmt " str:%s", value, "hello"); \ test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:00000123\r\n$9\r\nstr:hello\r\n",len) == 0 && \ len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ + hi_free(cmd); \ } while(0) #define FLOAT_WIDTH_TEST(type) do { \ @@ -228,7 +306,7 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"key:%08.3f str:%s", value, "hello"); \ test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:0123.000\r\n$9\r\nstr:hello\r\n",len) == 0 && \ len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ + hi_free(cmd); \ } while(0) INTEGER_WIDTH_TEST("d", int); @@ -259,29 +337,29 @@ static void test_format_commands(void) { len = redisFormatCommandArgv(&cmd,argc,argv,NULL); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command by passing argc/argv with lengths: "); len = redisFormatCommandArgv(&cmd,argc,argv,lens); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(7+2)+4+(3+2)); - free(cmd); + hi_free(cmd); - sds sds_cmd; + hisds sds_cmd; - sds_cmd = sdsempty(); - test("Format command into sds by passing argc/argv without lengths: "); + sds_cmd = NULL; + test("Format command into hisds by passing argc/argv without lengths: "); len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL); test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - sdsfree(sds_cmd); + hi_sdsfree(sds_cmd); - sds_cmd = sdsempty(); - test("Format command into sds by passing argc/argv with lengths: "); + sds_cmd = NULL; + test("Format command into hisds by passing argc/argv with lengths: "); len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens); test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(7+2)+4+(3+2)); - sdsfree(sds_cmd); + hi_sdsfree(sds_cmd); } static void test_append_formatted_commands(struct config config) { @@ -300,7 +378,7 @@ static void test_append_formatted_commands(struct config config) { assert(redisGetReply(c, (void*)&reply) == REDIS_OK); - free(cmd); + hi_free(cmd); freeReplyObject(reply); disconnect(c, 0); @@ -308,7 +386,7 @@ static void test_append_formatted_commands(struct config config) { static void test_reply_reader(void) { redisReader *reader; - void *reply; + void *reply, *root; int ret; int i; @@ -332,16 +410,26 @@ static void test_reply_reader(void) { strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); redisReaderFree(reader); - test("Set error on nested multi bulks with depth > 7: "); reader = redisReaderCreate(); - - for (i = 0; i < 9; i++) { - redisReaderFeed(reader,(char*)"*1\r\n",4); + test("Can handle arbitrarily nested multi-bulks: "); + for (i = 0; i < 128; i++) { + redisReaderFeed(reader,(char*)"*1\r\n", 4); } + redisReaderFeed(reader,(char*)"$6\r\nLOLWUT\r\n",12); + ret = redisReaderGetReply(reader,&reply); + root = reply; /* Keep track of the root reply */ + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_ARRAY && + ((redisReply*)reply)->elements == 1); - ret = redisReaderGetReply(reader,NULL); - test_cond(ret == REDIS_ERR && - strncasecmp(reader->errstr,"No support for",14) == 0); + test("Can parse arbitrarily nested multi-bulks correctly: "); + while(i--) { + assert(reply != NULL && ((redisReply*)reply)->type == REDIS_REPLY_ARRAY); + reply = ((redisReply*)reply)->element[0]; + } + test_cond(((redisReply*)reply)->type == REDIS_REPLY_STRING && + !memcmp(((redisReply*)reply)->str, "LOLWUT", 6)); + freeReplyObject(root); redisReaderFree(reader); test("Correctly parses LLONG_MAX: "); @@ -400,6 +488,16 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); + test("Can configure maximum multi-bulk elements: "); + reader = redisReaderCreate(); + reader->maxelements = 1024; + redisReaderFeed(reader, "*1025\r\n", 7); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr, "Multi-bulk length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + #if LLONG_MAX > SIZE_MAX test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); @@ -459,6 +557,32 @@ static void test_reply_reader(void) { ((redisReply*)reply)->elements == 0); freeReplyObject(reply); redisReaderFree(reader); + + /* RESP3 verbatim strings (GitHub issue #802) */ + test("Can parse RESP3 verbatim strings: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)"=10\r\ntxt:LOLWUT\r\n",17); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_VERB && + !memcmp(((redisReply*)reply)->str,"LOLWUT", 6)); + freeReplyObject(reply); + redisReaderFree(reader); + + /* RESP3 push messages (Github issue #815) */ + test("Can parse RESP3 push messages: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)">2\r\n$6\r\nLOLWUT\r\n:42\r\n",21); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_PUSH && + ((redisReply*)reply)->elements == 2 && + ((redisReply*)reply)->element[0]->type == REDIS_REPLY_STRING && + !memcmp(((redisReply*)reply)->element[0]->str,"LOLWUT",6) && + ((redisReply*)reply)->element[1]->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->element[1]->integer == 42); + freeReplyObject(reply); + redisReaderFree(reader); } static void test_free_null(void) { @@ -474,6 +598,47 @@ static void test_free_null(void) { test_cond(reply == NULL); } +static void *hi_malloc_fail(size_t size) { + (void)size; + return NULL; +} + +static void *hi_calloc_fail(size_t nmemb, size_t size) { + (void)nmemb; + (void)size; + return NULL; +} + +static void *hi_realloc_fail(void *ptr, size_t size) { + (void)ptr; + (void)size; + return NULL; +} + +static void test_allocator_injection(void) { + hiredisAllocFuncs ha = { + .mallocFn = hi_malloc_fail, + .callocFn = hi_calloc_fail, + .reallocFn = hi_realloc_fail, + .strdupFn = strdup, + .freeFn = free, + }; + + // Override hiredis allocators + hiredisSetAllocators(&ha); + + test("redisContext uses injected allocators: "); + redisContext *c = redisConnect("localhost", 6379); + test_cond(c == NULL); + + test("redisReader uses injected allocators: "); + redisReader *reader = redisReaderCreate(); + test_cond(reader == NULL); + + // Return allocators to default + hiredisResetAllocators(); +} + #define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; @@ -491,19 +656,19 @@ static void test_blocking_connection_errors(void) { (strcmp(c->errstr, "Name or service not known") == 0 || strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || strcmp(c->errstr, "Name does not resolve") == 0 || - strcmp(c->errstr, - "nodename nor servname provided, or not known") == 0 || + strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || strcmp(c->errstr, "No address associated with hostname") == 0 || strcmp(c->errstr, "Temporary failure in name resolution") == 0 || - strcmp(c->errstr, - "hostname nor servname provided, or not known") == 0 || - strcmp(c->errstr, "no address associated with name") == 0)); + strcmp(c->errstr, "hostname nor servname provided, or not known") == 0 || + strcmp(c->errstr, "no address associated with name") == 0 || + strcmp(c->errstr, "No such host is known. ") == 0)); redisFree(c); } else { printf("Skipping NXDOMAIN test. Found evil ISP!\n"); freeaddrinfo(ai_tmp); } +#ifndef _WIN32 test("Returns error when the port is not open: "); c = redisConnect((char*)"localhost", 1); test_cond(c->err == REDIS_ERR_IO && @@ -514,11 +679,166 @@ static void test_blocking_connection_errors(void) { c = redisConnectUnix((char*)"/tmp/idontexist.sock"); test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */ redisFree(c); +#endif +} + +/* Test push handler */ +void push_handler(void *privdata, void *r) { + struct pushCounters *pcounts = privdata; + redisReply *reply = r, *payload; + + assert(reply && reply->type == REDIS_REPLY_PUSH && reply->elements == 2); + + payload = reply->element[1]; + if (payload->type == REDIS_REPLY_ARRAY) { + payload = payload->element[0]; + } + + if (payload->type == REDIS_REPLY_STRING) { + pcounts->str++; + } else if (payload->type == REDIS_REPLY_NIL) { + pcounts->nil++; + } + + freeReplyObject(reply); +} + +/* Dummy function just to test setting a callback with redisOptions */ +void push_handler_async(redisAsyncContext *ac, void *reply) { + (void)ac; + (void)reply; +} + +static void test_resp3_push_handler(redisContext *c) { + struct pushCounters pc = {0}; + redisPushFn *old = NULL; + redisReply *reply; + void *privdata; + + /* Switch to RESP3 and turn on client tracking */ + send_hello(c, 3); + send_client_tracking(c, "ON"); + privdata = c->privdata; + c->privdata = &pc; + + reply = redisCommand(c, "GET key:0"); + assert(reply != NULL); + freeReplyObject(reply); + + test("RESP3 PUSH messages are handled out of band by default: "); + reply = redisCommand(c, "SET key:0 val:0"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + + assert((reply = redisCommand(c, "GET key:0")) != NULL); + freeReplyObject(reply); + + old = redisSetPushCallback(c, push_handler); + test("We can set a custom RESP3 PUSH handler: "); + reply = redisCommand(c, "SET key:0 val:0"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.str == 1); + freeReplyObject(reply); + + test("We properly handle a NIL invalidation payload: "); + reply = redisCommand(c, "FLUSHDB"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.nil == 1); + freeReplyObject(reply); + + /* Unset the push callback and generate an invalidate message making + * sure it is not handled out of band. */ + test("With no handler, PUSH replies come in-band: "); + redisSetPushCallback(c, NULL); + assert((reply = redisCommand(c, "GET key:0")) != NULL); + freeReplyObject(reply); + assert((reply = redisCommand(c, "SET key:0 invalid")) != NULL); + test_cond(reply->type == REDIS_REPLY_PUSH); + freeReplyObject(reply); + + test("With no PUSH handler, no replies are lost: "); + assert(redisGetReply(c, (void**)&reply) == REDIS_OK); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + + /* Return to the originally set PUSH handler */ + assert(old != NULL); + redisSetPushCallback(c, old); + + /* Switch back to RESP2 and disable tracking */ + c->privdata = privdata; + send_client_tracking(c, "OFF"); + send_hello(c, 2); +} + +redisOptions get_redis_tcp_options(struct config config) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, config.tcp.host, config.tcp.port); + return options; +} + +static void test_resp3_push_options(struct config config) { + redisAsyncContext *ac; + redisContext *c; + redisOptions options; + + test("We set a default RESP3 handler for redisContext: "); + options = get_redis_tcp_options(config); + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb != NULL); + redisFree(c); + + test("We don't set a default RESP3 push handler for redisAsyncContext: "); + options = get_redis_tcp_options(config); + assert((ac = redisAsyncConnectWithOptions(&options)) != NULL); + test_cond(ac->c.push_cb == NULL); + redisAsyncFree(ac); + + test("Our REDIS_OPT_NO_PUSH_AUTOFREE flag works: "); + options = get_redis_tcp_options(config); + options.options |= REDIS_OPT_NO_PUSH_AUTOFREE; + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb == NULL); + redisFree(c); + + test("We can use redisOptions to set a custom PUSH handler for redisContext: "); + options = get_redis_tcp_options(config); + options.push_cb = push_handler; + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb == push_handler); + redisFree(c); + + test("We can use redisOptions to set a custom PUSH handler for redisAsyncContext: "); + options = get_redis_tcp_options(config); + options.async_push_cb = push_handler_async; + assert((ac = redisAsyncConnectWithOptions(&options)) != NULL); + test_cond(ac->push_cb == push_handler_async); + redisAsyncFree(ac); +} + +void free_privdata(void *privdata) { + struct privdata *data = privdata; + data->dtor_counter++; +} + +static void test_privdata_hooks(struct config config) { + struct privdata data = {0}; + redisOptions options; + redisContext *c; + + test("We can use redisOptions to set privdata: "); + options = get_redis_tcp_options(config); + REDIS_OPTIONS_SET_PRIVDATA(&options, &data, free_privdata); + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->privdata == &data); + + test("Our privdata destructor fires when we free the context: "); + redisFree(c); + test_cond(data.dtor_counter == 1); } static void test_blocking_connection(struct config config) { redisContext *c; redisReply *reply; + int major; c = do_connect(config); @@ -591,14 +911,42 @@ static void test_blocking_connection(struct config config) { strcasecmp(reply->element[1]->str,"pong") == 0); freeReplyObject(reply); + /* Make sure passing NULL to redisGetReply is safe */ + test("Can pass NULL to redisGetReply: "); + assert(redisAppendCommand(c, "PING") == REDIS_OK); + test_cond(redisGetReply(c, NULL) == REDIS_OK); + + get_redis_version(c, &major, NULL); + if (major >= 6) test_resp3_push_handler(c); + test_resp3_push_options(config); + + test_privdata_hooks(config); + disconnect(c, 0); } +/* Send DEBUG SLEEP 0 to detect if we have this command */ +static int detect_debug_sleep(redisContext *c) { + int detected; + redisReply *reply = redisCommand(c, "DEBUG SLEEP 0\r\n"); + + if (reply == NULL || c->err) { + const char *cause = c->err ? c->errstr : "(none)"; + fprintf(stderr, "Error testing for DEBUG SLEEP (Redis error: %s), exiting\n", cause); + exit(-1); + } + + detected = reply->type == REDIS_REPLY_STATUS; + freeReplyObject(reply); + + return detected; +} + static void test_blocking_connection_timeouts(struct config config) { redisContext *c; redisReply *reply; ssize_t s; - const char *cmd = "DEBUG SLEEP 3\r\n"; + const char *sleep_cmd = "DEBUG SLEEP 3\r\n"; struct timeval tv; c = do_connect(config); @@ -615,14 +963,24 @@ static void test_blocking_connection_timeouts(struct config config) { c = do_connect(config); test("Does not return a reply when the command times out: "); - redisAppendFormattedCommand(c, cmd, strlen(cmd)); - s = c->funcs->write(c); - tv.tv_sec = 0; - tv.tv_usec = 10000; - redisSetTimeout(c, tv); - reply = redisCommand(c, "GET foo"); - test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0); - freeReplyObject(reply); + if (detect_debug_sleep(c)) { + redisAppendFormattedCommand(c, sleep_cmd, strlen(sleep_cmd)); + s = c->funcs->write(c); + tv.tv_sec = 0; + tv.tv_usec = 10000; + redisSetTimeout(c, tv); + reply = redisCommand(c, "GET foo"); +#ifndef _WIN32 + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && + strcmp(c->errstr, "Resource temporarily unavailable") == 0); +#else + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_TIMEOUT && + strcmp(c->errstr, "recv timeout") == 0); +#endif + freeReplyObject(reply); + } else { + test_skipped(); + } test("Reconnect properly reconnects after a timeout: "); do_reconnect(c, config); @@ -649,18 +1007,7 @@ static void test_blocking_io_errors(struct config config) { /* Connect to target given by config. */ c = do_connect(config); - { - /* Find out Redis version to determine the path for the next test */ - const char *field = "redis_version:"; - char *p, *eptr; - - reply = redisCommand(c,"INFO"); - p = strstr(reply->str,field); - major = strtol(p+strlen(field),&eptr,10); - p = eptr+1; /* char next to the first "." */ - minor = strtol(p,&eptr,10); - freeReplyObject(reply); - } + get_redis_version(c, &major, &minor); test("Returns I/O error when the connection is lost: "); reply = redisCommand(c,"QUIT"); @@ -674,6 +1021,7 @@ static void test_blocking_io_errors(struct config config) { test_cond(reply == NULL); } +#ifndef _WIN32 /* On 2.0, QUIT will cause the connection to be closed immediately and * the read(2) for the reply on QUIT will set the error to EOF. * On >2.0, QUIT will return with OK and another read(2) needed to be @@ -681,14 +1029,19 @@ static void test_blocking_io_errors(struct config config) { * conditions, the error will be set to EOF. */ assert(c->err == REDIS_ERR_EOF && strcmp(c->errstr,"Server closed the connection") == 0); +#endif redisFree(c); c = do_connect(config); test("Returns I/O error on socket timeout: "); struct timeval tv = { 0, 1000 }; assert(redisSetTimeout(c,tv) == REDIS_OK); - test_cond(redisGetReply(c,&_reply) == REDIS_ERR && - c->err == REDIS_ERR_IO && errno == EAGAIN); + int respcode = redisGetReply(c,&_reply); +#ifndef _WIN32 + test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_IO && errno == EAGAIN); +#else + test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_TIMEOUT); +#endif redisFree(c); } @@ -716,6 +1069,18 @@ static void test_invalid_timeout_errors(struct config config) { redisFree(c); } +/* Wrap malloc to abort on failure so OOM checks don't make the test logic + * harder to follow. */ +void *hi_malloc_safe(size_t size) { + void *ptr = hi_malloc(size); + if (ptr == NULL) { + fprintf(stderr, "Error: Out of memory\n"); + exit(-1); + } + + return ptr; +} + static void test_throughput(struct config config) { redisContext *c = do_connect(config); redisReply **replies; @@ -727,7 +1092,7 @@ static void test_throughput(struct config config) { freeReplyObject(redisCommand(c,"LPUSH mylist foo")); num = 1000; - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c,"PING"); @@ -735,10 +1100,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx PING: %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c,"LRANGE mylist 0 499"); @@ -747,10 +1112,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c, "INCRBY incrkey %d", 1000000); @@ -758,11 +1123,11 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx INCRBY: %.3fs)\n", num, (t2-t1)/1000000.0); num = 10000; - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"PING"); t1 = usec(); @@ -772,10 +1137,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx PING (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"LRANGE mylist 0 499"); t1 = usec(); @@ -786,10 +1151,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"INCRBY incrkey %d", 1000000); t1 = usec(); @@ -799,7 +1164,7 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx INCRBY (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); disconnect(c, 0); @@ -916,9 +1281,8 @@ int main(int argc, char **argv) { }; int throughput = 1; int test_inherit_fd = 1; - - /* Ignore broken pipe signal (for I/O error tests). */ - signal(SIGPIPE, SIG_IGN); + int skips_as_fails = 0; + int test_unix_socket; /* Parse command line options. */ argv++; argc--; @@ -936,6 +1300,8 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; + } else if (argc >= 1 && !strcmp(argv[0],"--skips-as-fails")) { + skips_as_fails = 1; #ifdef HIREDIS_TEST_SSL } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { argv++; argc--; @@ -960,6 +1326,19 @@ int main(int argc, char **argv) { argv++; argc--; } +#ifndef _WIN32 + /* Ignore broken pipe signal (for I/O error tests). */ + signal(SIGPIPE, SIG_IGN); + + test_unix_socket = access(cfg.unix_sock.path, F_OK) == 0; + +#else + /* Unix sockets don't exist in Windows */ + test_unix_socket = 0; +#endif + + test_allocator_injection(); + test_format_commands(); test_reply_reader(); test_blocking_connection_errors(); @@ -974,15 +1353,25 @@ int main(int argc, char **argv) { test_append_formatted_commands(cfg); if (throughput) test_throughput(cfg); - printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path); - cfg.type = CONN_UNIX; - test_blocking_connection(cfg); - test_blocking_connection_timeouts(cfg); - test_blocking_io_errors(cfg); - if (throughput) test_throughput(cfg); + printf("\nTesting against Unix socket connection (%s): ", cfg.unix_sock.path); + if (test_unix_socket) { + printf("\n"); + cfg.type = CONN_UNIX; + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + if (throughput) test_throughput(cfg); + } else { + test_skipped(); + } #ifdef HIREDIS_TEST_SSL if (cfg.ssl.port && cfg.ssl.host) { + + redisInitOpenSSL(); + _ssl_ctx = redisCreateSSLContext(cfg.ssl.ca_cert, NULL, cfg.ssl.cert, cfg.ssl.key, NULL, NULL); + assert(_ssl_ctx != NULL); + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); cfg.type = CONN_SSL; @@ -992,21 +1381,31 @@ int main(int argc, char **argv) { test_invalid_timeout_errors(cfg); test_append_formatted_commands(cfg); if (throughput) test_throughput(cfg); + + redisFreeSSLContext(_ssl_ctx); + _ssl_ctx = NULL; } #endif if (test_inherit_fd) { - printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); - cfg.type = CONN_FD; - test_blocking_connection(cfg); + printf("\nTesting against inherited fd (%s): ", cfg.unix_sock.path); + if (test_unix_socket) { + printf("\n"); + cfg.type = CONN_FD; + test_blocking_connection(cfg); + } else { + test_skipped(); + } } - - if (fails) { + if (fails || (skips_as_fails && skips)) { printf("*** %d TESTS FAILED ***\n", fails); + if (skips) { + printf("*** %d TESTS SKIPPED ***\n", skips); + } return 1; } - printf("ALL TESTS PASSED\n"); + printf("ALL TESTS PASSED (%d skipped)\n", skips); return 0; } diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh index 2cab9e6fb..c72bcb0dc 100755 --- a/deps/hiredis/test.sh +++ b/deps/hiredis/test.sh @@ -4,7 +4,9 @@ REDIS_SERVER=${REDIS_SERVER:-redis-server} REDIS_PORT=${REDIS_PORT:-56379} REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} TEST_SSL=${TEST_SSL:-0} +SKIPS_AS_FAILS=${SKIPS_AS_FAILS-:0} SSL_TEST_ARGS= +SKIPS_ARG= tmpdir=$(mktemp -d) PID_FILE=${tmpdir}/hiredis-test-redis.pid @@ -67,4 +69,10 @@ fi cat ${tmpdir}/redis.conf ${REDIS_SERVER} ${tmpdir}/redis.conf -${TEST_PREFIX:-} ./hiredis-test -h 127.0.0.1 -p ${REDIS_PORT} -s ${SOCK_FILE} ${SSL_TEST_ARGS} +# Wait until we detect the unix socket +while [ ! -S "${SOCK_FILE}" ]; do sleep 1; done + +# Treat skips as failures if directed +[ "$SKIPS_AS_FAILS" = 1 ] && SKIPS_ARG="--skips-as-fails" + +${TEST_PREFIX:-} ./hiredis-test -h 127.0.0.1 -p ${REDIS_PORT} -s ${SOCK_FILE} ${SSL_TEST_ARGS} ${SKIPS_ARG} diff --git a/deps/jemalloc/.appveyor.yml b/deps/jemalloc/.appveyor.yml index 9a7d00a99..90b03688d 100644 --- a/deps/jemalloc/.appveyor.yml +++ b/deps/jemalloc/.appveyor.yml @@ -5,27 +5,27 @@ environment: - MSYSTEM: MINGW64 CPU: x86_64 MSVC: amd64 + CONFIG_FLAGS: --enable-debug + - MSYSTEM: MINGW64 + CPU: x86_64 + CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW32 CPU: i686 MSVC: x86 - - MSYSTEM: MINGW64 - CPU: x86_64 + CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW32 CPU: i686 + CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW64 CPU: x86_64 MSVC: amd64 - CONFIG_FLAGS: --enable-debug + - MSYSTEM: MINGW64 + CPU: x86_64 - MSYSTEM: MINGW32 CPU: i686 MSVC: x86 - CONFIG_FLAGS: --enable-debug - - MSYSTEM: MINGW64 - CPU: x86_64 - CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW32 CPU: i686 - CONFIG_FLAGS: --enable-debug install: - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH% diff --git a/deps/jemalloc/.cirrus.yml b/deps/jemalloc/.cirrus.yml new file mode 100644 index 000000000..019d2c38c --- /dev/null +++ b/deps/jemalloc/.cirrus.yml @@ -0,0 +1,21 @@ +env: + CIRRUS_CLONE_DEPTH: 1 + ARCH: amd64 + +task: + freebsd_instance: + matrix: + image: freebsd-12-0-release-amd64 + image: freebsd-11-2-release-amd64 + install_script: + - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf + - pkg upgrade -y + - pkg install -y autoconf gmake + script: + - autoconf + #- ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS + - ./configure + - export JFLAG=`sysctl -n kern.smp.cpus` + - gmake -j${JFLAG} + - gmake -j${JFLAG} tests + - gmake check diff --git a/deps/jemalloc/.gitignore b/deps/jemalloc/.gitignore index 19199ccb7..5ca0ad1da 100644 --- a/deps/jemalloc/.gitignore +++ b/deps/jemalloc/.gitignore @@ -30,7 +30,6 @@ /include/jemalloc/internal/public_namespace.h /include/jemalloc/internal/public_symbols.txt /include/jemalloc/internal/public_unnamespace.h -/include/jemalloc/internal/size_classes.h /include/jemalloc/jemalloc.h /include/jemalloc/jemalloc_defs.h /include/jemalloc/jemalloc_macros.h diff --git a/deps/jemalloc/.travis.yml b/deps/jemalloc/.travis.yml index 4cc116e5f..2da5da8eb 100644 --- a/deps/jemalloc/.travis.yml +++ b/deps/jemalloc/.travis.yml @@ -11,7 +11,7 @@ matrix: env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: + addons: &gcc_multilib apt: packages: - gcc-multilib @@ -21,6 +21,10 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -37,20 +41,25 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: osx + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: osx + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -61,50 +70,39 @@ matrix: env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + addons: *gcc_multilib + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -115,6 +113,10 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -123,6 +125,10 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -131,6 +137,24 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false,dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -143,10 +167,25 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + # Development build + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + # --enable-expermental-smallocx: + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + + # Valgrind + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" + addons: + apt: + packages: + - valgrind before_script: - autoconf + - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script - ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS - make -j3 - make -j3 tests diff --git a/deps/jemalloc/COPYING b/deps/jemalloc/COPYING index 98458d971..3b7fd3585 100644 --- a/deps/jemalloc/COPYING +++ b/deps/jemalloc/COPYING @@ -1,10 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2018 Jason Evans . +Copyright (C) 2002-present Jason Evans . All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2018 Facebook, Inc. All rights reserved. +Copyright (C) 2009-present Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/deps/jemalloc/ChangeLog b/deps/jemalloc/ChangeLog index 29a00fb78..e55813b7b 100644 --- a/deps/jemalloc/ChangeLog +++ b/deps/jemalloc/ChangeLog @@ -4,7 +4,143 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc -* 5.1.0 (May 4th, 2018) +* 5.2.1 (August 5, 2019) + + This release is primarily about Windows. A critical virtual memory leak is + resolved on all Windows platforms. The regression was present in all releases + since 5.0.0. + + Bug fixes: + - Fix a severe virtual memory leak on Windows. This regression was first + released in 5.0.0. (@Ignition, @j0t, @frederik-h, @davidtgoldblatt, + @interwq) + - Fix size 0 handling in posix_memalign(). This regression was first released + in 5.2.0. (@interwq) + - Fix the prof_log unit test which may observe unexpected backtraces from + compiler optimizations. The test was first added in 5.2.0. (@marxin, + @gnzlbg, @interwq) + - Fix the declaration of the extent_avail tree. This regression was first + released in 5.1.0. (@zoulasc) + - Fix an incorrect reference in jeprof. This functionality was first released + in 3.0.0. (@prehistoric-penguin) + - Fix an assertion on the deallocation fast-path. This regression was first + released in 5.2.0. (@yinan1048576) + - Fix the TLS_MODEL attribute in headers. This regression was first released + in 5.0.0. (@zoulasc, @interwq) + + Optimizations and refactors: + - Implement opt.retain on Windows and enable by default on 64-bit. (@interwq, + @davidtgoldblatt) + - Optimize away a branch on the operator delete[] path. (@mgrice) + - Add format annotation to the format generator function. (@zoulasc) + - Refactor and improve the size class header generation. (@yinan1048576) + - Remove best fit. (@djwatson) + - Avoid blocking on background thread locks for stats. (@oranagra, @interwq) + +* 5.2.0 (April 2, 2019) + + This release includes a few notable improvements, which are summarized below: + 1) improved fast-path performance from the optimizations by @djwatson; 2) + reduced virtual memory fragmentation and metadata usage; and 3) bug fixes on + setting the number of background threads. In addition, peak / spike memory + usage is improved with certain allocation patterns. As usual, the release and + prior dev versions have gone through large-scale production testing. + + New features: + - Implement oversize_threshold, which uses a dedicated arena for allocations + crossing the specified threshold to reduce fragmentation. (@interwq) + - Add extents usage information to stats. (@tyleretzel) + - Log time information for sampled allocations. (@tyleretzel) + - Support 0 size in sdallocx. (@djwatson) + - Output rate for certain counters in malloc_stats. (@zinoale) + - Add configure option --enable-readlinkat, which allows the use of readlinkat + over readlink. (@davidtgoldblatt) + - Add configure options --{enable,disable}-{static,shared} to allow not + building unwanted libraries. (@Ericson2314) + - Add configure option --disable-libdl to enable fully static builds. + (@interwq) + - Add mallctl interfaces: + + opt.oversize_threshold (@interwq) + + stats.arenas..extent_avail (@tyleretzel) + + stats.arenas..extents..n{dirty,muzzy,retained} (@tyleretzel) + + stats.arenas..extents..{dirty,muzzy,retained}_bytes + (@tyleretzel) + + Portability improvements: + - Update MSVC builds. (@maksqwe, @rustyx) + - Workaround a compiler optimizer bug on s390x. (@rkmisra) + - Make use of pthread_set_name_np(3) on FreeBSD. (@trasz) + - Implement malloc_getcpu() to enable percpu_arena for windows. (@santagada) + - Link against -pthread instead of -lpthread. (@paravoid) + - Make background_thread not dependent on libdl. (@interwq) + - Add stringify to fix a linker directive issue on MSVC. (@daverigby) + - Detect and fall back when 8-bit atomics are unavailable. (@interwq) + - Fall back to the default pthread_create if dlsym(3) fails. (@interwq) + + Optimizations and refactors: + - Refactor the TSD module. (@davidtgoldblatt) + - Avoid taking extents_muzzy mutex when muzzy is disabled. (@interwq) + - Avoid taking large_mtx for auto arenas on the tcache flush path. (@interwq) + - Optimize ixalloc by avoiding a size lookup. (@interwq) + - Implement opt.oversize_threshold which uses a dedicated arena for requests + crossing the threshold, also eagerly purges the oversize extents. Default + the threshold to 8 MiB. (@interwq) + - Clean compilation with -Wextra. (@gnzlbg, @jasone) + - Refactor the size class module. (@davidtgoldblatt) + - Refactor the stats emitter. (@tyleretzel) + - Optimize pow2_ceil. (@rkmisra) + - Avoid runtime detection of lazy purging on FreeBSD. (@trasz) + - Optimize mmap(2) alignment handling on FreeBSD. (@trasz) + - Improve error handling for THP state initialization. (@jsteemann) + - Rework the malloc() fast path. (@djwatson) + - Rework the free() fast path. (@djwatson) + - Refactor and optimize the tcache fill / flush paths. (@djwatson) + - Optimize sync / lwsync on PowerPC. (@chmeeedalf) + - Bypass extent_dalloc() when retain is enabled. (@interwq) + - Optimize the locking on large deallocation. (@interwq) + - Reduce the number of pages committed from sanity checking in debug build. + (@trasz, @interwq) + - Deprecate OSSpinLock. (@interwq) + - Lower the default number of background threads to 4 (when the feature + is enabled). (@interwq) + - Optimize the trylock spin wait. (@djwatson) + - Use arena index for arena-matching checks. (@interwq) + - Avoid forced decay on thread termination when using background threads. + (@interwq) + - Disable muzzy decay by default. (@djwatson, @interwq) + - Only initialize libgcc unwinder when profiling is enabled. (@paravoid, + @interwq) + + Bug fixes (all only relevant to jemalloc 5.x): + - Fix background thread index issues with max_background_threads. (@djwatson, + @interwq) + - Fix stats output for opt.lg_extent_max_active_fit. (@interwq) + - Fix opt.prof_prefix initialization. (@davidtgoldblatt) + - Properly trigger decay on tcache destroy. (@interwq, @amosbird) + - Fix tcache.flush. (@interwq) + - Detect whether explicit extent zero out is necessary with huge pages or + custom extent hooks, which may change the purge semantics. (@interwq) + - Fix a side effect caused by extent_max_active_fit combined with decay-based + purging, where freed extents can accumulate and not be reused for an + extended period of time. (@interwq, @mpghf) + - Fix a missing unlock on extent register error handling. (@zoulasc) + + Testing: + - Simplify the Travis script output. (@gnzlbg) + - Update the test scripts for FreeBSD. (@devnexen) + - Add unit tests for the producer-consumer pattern. (@interwq) + - Add Cirrus-CI config for FreeBSD builds. (@jasone) + - Add size-matching sanity checks on tcache flush. (@davidtgoldblatt, + @interwq) + + Incompatible changes: + - Remove --with-lg-page-sizes. (@davidtgoldblatt) + + Documentation: + - Attempt to build docs by default, however skip doc building when xsltproc + is missing. (@interwq, @cmuellner) + +* 5.1.0 (May 4, 2018) This release is primarily about fine-tuning, ranging from several new features to numerous notable performance and portability enhancements. The release and diff --git a/deps/jemalloc/INSTALL.md b/deps/jemalloc/INSTALL.md index ef328c60f..b8f729b0d 100644 --- a/deps/jemalloc/INSTALL.md +++ b/deps/jemalloc/INSTALL.md @@ -221,13 +221,6 @@ any of the following arguments (not a definitive list) to 'configure': system page size may change between configuration and execution, e.g. when cross compiling. -* `--with-lg-page-sizes=` - - Specify the comma-separated base 2 logs of the page sizes to support. This - option may be useful when cross compiling in combination with - `--with-lg-page`, but its primary use case is for integration with FreeBSD's - libc, wherein jemalloc is embedded. - * `--with-lg-hugepage=` Specify the base 2 log of the system huge page size. This option is useful @@ -276,6 +269,11 @@ any of the following arguments (not a definitive list) to 'configure': in the same process, which will almost certainly result in confusing runtime crashes if pointers leak from one implementation to the other. +* `--disable-libdl` + + Disable the usage of libdl, namely dlsym(3) which is required by the lazy + lock option. This can allow building static binaries. + The following environment variables (not a definitive list) impact configure's behavior: diff --git a/deps/jemalloc/Makefile.in b/deps/jemalloc/Makefile.in index 9b9347fff..7128b007e 100644 --- a/deps/jemalloc/Makefile.in +++ b/deps/jemalloc/Makefile.in @@ -47,6 +47,7 @@ REV := @rev@ install_suffix := @install_suffix@ ABI := @abi@ XSLTPROC := @XSLTPROC@ +XSLROOT := @XSLROOT@ AUTOCONF := @AUTOCONF@ _RPATH = @RPATH@ RPATH = $(if $(1),$(call _RPATH,$(1))) @@ -55,8 +56,12 @@ cfghdrs_out := @cfghdrs_out@ cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@) cfgoutputs_out := @cfgoutputs_out@ enable_autogen := @enable_autogen@ +enable_doc := @enable_doc@ +enable_shared := @enable_shared@ +enable_static := @enable_static@ enable_prof := @enable_prof@ enable_zone_allocator := @enable_zone_allocator@ +enable_experimental_smallocx := @enable_experimental_smallocx@ MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF link_whole_archive := @link_whole_archive@ DSO_LDFLAGS = @DSO_LDFLAGS@ @@ -102,7 +107,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/extent_dss.c \ $(srcroot)src/extent_mmap.c \ $(srcroot)src/hash.c \ - $(srcroot)src/hooks.c \ + $(srcroot)src/hook.c \ $(srcroot)src/large.c \ $(srcroot)src/log.c \ $(srcroot)src/malloc_io.c \ @@ -113,9 +118,12 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/prng.c \ $(srcroot)src/prof.c \ $(srcroot)src/rtree.c \ + $(srcroot)src/safety_check.c \ $(srcroot)src/stats.c \ + $(srcroot)src/sc.c \ $(srcroot)src/sz.c \ $(srcroot)src/tcache.c \ + $(srcroot)src/test_hooks.c \ $(srcroot)src/ticker.c \ $(srcroot)src/tsd.c \ $(srcroot)src/witness.c @@ -165,14 +173,18 @@ TESTS_UNIT := \ $(srcroot)test/unit/background_thread_enable.c \ $(srcroot)test/unit/base.c \ $(srcroot)test/unit/bitmap.c \ + $(srcroot)test/unit/bit_util.c \ + $(srcroot)test/unit/binshard.c \ $(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/decay.c \ $(srcroot)test/unit/div.c \ $(srcroot)test/unit/emitter.c \ $(srcroot)test/unit/extent_quantize.c \ + $(srcroot)test/unit/extent_util.c \ $(srcroot)test/unit/fork.c \ $(srcroot)test/unit/hash.c \ - $(srcroot)test/unit/hooks.c \ + $(srcroot)test/unit/hook.c \ + $(srcroot)test/unit/huge.c \ $(srcroot)test/unit/junk.c \ $(srcroot)test/unit/junk_alloc.c \ $(srcroot)test/unit/junk_free.c \ @@ -190,6 +202,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/prof_active.c \ $(srcroot)test/unit/prof_gdump.c \ $(srcroot)test/unit/prof_idump.c \ + $(srcroot)test/unit/prof_log.c \ $(srcroot)test/unit/prof_reset.c \ $(srcroot)test/unit/prof_tctx.c \ $(srcroot)test/unit/prof_thread_name.c \ @@ -198,13 +211,17 @@ TESTS_UNIT := \ $(srcroot)test/unit/rb.c \ $(srcroot)test/unit/retained.c \ $(srcroot)test/unit/rtree.c \ + $(srcroot)test/unit/safety_check.c \ + $(srcroot)test/unit/seq.c \ $(srcroot)test/unit/SFMT.c \ + $(srcroot)test/unit/sc.c \ $(srcroot)test/unit/size_classes.c \ $(srcroot)test/unit/slab.c \ $(srcroot)test/unit/smoothstep.c \ $(srcroot)test/unit/spin.c \ $(srcroot)test/unit/stats.c \ $(srcroot)test/unit/stats_print.c \ + $(srcroot)test/unit/test_hooks.c \ $(srcroot)test/unit/ticker.c \ $(srcroot)test/unit/nstime.c \ $(srcroot)test/unit/tsd.c \ @@ -217,15 +234,21 @@ endif TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/allocated.c \ $(srcroot)test/integration/extent.c \ + $(srcroot)test/integration/malloc.c \ $(srcroot)test/integration/mallocx.c \ $(srcroot)test/integration/MALLOCX_ARENA.c \ $(srcroot)test/integration/overflow.c \ $(srcroot)test/integration/posix_memalign.c \ $(srcroot)test/integration/rallocx.c \ $(srcroot)test/integration/sdallocx.c \ + $(srcroot)test/integration/slab_sizes.c \ $(srcroot)test/integration/thread_arena.c \ $(srcroot)test/integration/thread_tcache_enabled.c \ $(srcroot)test/integration/xallocx.c +ifeq (@enable_experimental_smallocx@, 1) +TESTS_INTEGRATION += \ + $(srcroot)test/integration/smallocx.c +endif ifeq (@enable_cxx@, 1) CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp @@ -233,7 +256,9 @@ else CPP_SRCS := TESTS_INTEGRATION_CPP := endif -TESTS_STRESS := $(srcroot)test/stress/microbench.c +TESTS_STRESS := $(srcroot)test/stress/microbench.c \ + $(srcroot)test/stress/hookbench.c + TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_INTEGRATION_CPP) $(TESTS_STRESS) @@ -274,10 +299,24 @@ all: build_lib dist: build_doc $(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl +ifneq ($(XSLROOT),) $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< +else +ifeq ($(wildcard $(DOCS_HTML)),) + @echo "

Missing xsltproc. Doc not built.

" > $@ +endif + @echo "Missing xsltproc. "$@" not (re)built." +endif $(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl +ifneq ($(XSLROOT),) $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< +else +ifeq ($(wildcard $(DOCS_MAN3)),) + @echo "Missing xsltproc. Doc not built." > $@ +endif + @echo "Missing xsltproc. "$@" not (re)built." +endif build_doc_html: $(DOCS_HTML) build_doc_man: $(DOCS_MAN3) @@ -400,7 +439,7 @@ $(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(C_JET_OBJS) $(C_TESTLI $(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) - $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -lpthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS) + $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -pthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS) $(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) @@ -412,7 +451,12 @@ $(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TE build_lib_shared: $(DSOS) build_lib_static: $(STATIC_LIBS) -build_lib: build_lib_shared build_lib_static +ifeq ($(enable_shared), 1) +build_lib: build_lib_shared +endif +ifeq ($(enable_static), 1) +build_lib: build_lib_static +endif install_bin: $(INSTALL) -d $(BINDIR) @@ -449,7 +493,13 @@ install_lib_pc: $(PC) $(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \ done -install_lib: install_lib_shared install_lib_static install_lib_pc +ifeq ($(enable_shared), 1) +install_lib: install_lib_shared +endif +ifeq ($(enable_static), 1) +install_lib: install_lib_static +endif +install_lib: install_lib_pc install_doc_html: $(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix) @@ -465,9 +515,13 @@ install_doc_man: $(INSTALL) -m 644 $$d $(MANDIR)/man3; \ done -install_doc: install_doc_html install_doc_man +install_doc: build_doc install_doc_html install_doc_man -install: install_bin install_include install_lib install_doc +install: install_bin install_include install_lib + +ifeq ($(enable_doc), 1) +install: install_doc +endif tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE)) tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE)) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%$(EXE)) diff --git a/deps/jemalloc/VERSION b/deps/jemalloc/VERSION index 5c2e26d43..05500db52 100644 --- a/deps/jemalloc/VERSION +++ b/deps/jemalloc/VERSION @@ -1 +1 @@ -5.1.0-0-g0 +5.2.1-0-g0 diff --git a/deps/jemalloc/bin/jeprof.in b/deps/jemalloc/bin/jeprof.in index 588c6b438..3ed408c9c 100644 --- a/deps/jemalloc/bin/jeprof.in +++ b/deps/jemalloc/bin/jeprof.in @@ -2909,6 +2909,7 @@ sub RemoveUninterestingFrames { '@JEMALLOC_PREFIX@xallocx', '@JEMALLOC_PREFIX@dallocx', '@JEMALLOC_PREFIX@sdallocx', + '@JEMALLOC_PREFIX@sdallocx_noflags', 'tc_calloc', 'tc_cfree', 'tc_malloc', @@ -5366,7 +5367,7 @@ sub GetProcedureBoundaries { my $demangle_flag = ""; my $cppfilt_flag = ""; my $to_devnull = ">$dev_null 2>&1"; - if (system(ShellEscape($nm, "--demangle", "image") . $to_devnull) == 0) { + if (system(ShellEscape($nm, "--demangle", $image) . $to_devnull) == 0) { # In this mode, we do "nm --demangle " $demangle_flag = "--demangle"; $cppfilt_flag = ""; diff --git a/deps/jemalloc/configure b/deps/jemalloc/configure index 6aebfad0d..dace1f69e 100755 --- a/deps/jemalloc/configure +++ b/deps/jemalloc/configure @@ -629,18 +629,16 @@ enable_initial_exec_tls enable_zone_allocator enable_tls enable_lazy_lock -jemalloc_version_gid -jemalloc_version_nrev -jemalloc_version_bugfix -jemalloc_version_minor -jemalloc_version_major -jemalloc_version +libdl +enable_opt_safety_checks +enable_readlinkat enable_log enable_cache_oblivious enable_xmalloc enable_utrace enable_fill enable_prof +enable_experimental_smallocx enable_stats enable_debug je_ @@ -648,6 +646,9 @@ install_suffix private_namespace JEMALLOC_CPREFIX JEMALLOC_PREFIX +enable_static +enable_shared +enable_doc AUTOCONF LD RANLIB @@ -679,6 +680,12 @@ so LD_PRELOAD_VAR RPATH abi +jemalloc_version_gid +jemalloc_version_nrev +jemalloc_version_bugfix +jemalloc_version_minor +jemalloc_version_major +jemalloc_version AWK NM AR @@ -744,6 +751,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -769,8 +777,12 @@ enable_option_checking with_xslroot enable_cxx with_lg_vaddr +with_version with_rpath enable_autogen +enable_doc +enable_shared +enable_static with_mangling with_jemalloc_prefix with_export @@ -779,6 +791,7 @@ with_install_suffix with_malloc_conf enable_debug enable_stats +enable_experimental_smallocx enable_prof enable_prof_libunwind with_static_libunwind @@ -789,11 +802,12 @@ enable_utrace enable_xmalloc enable_cache_oblivious enable_log +enable_readlinkat +enable_opt_safety_checks with_lg_quantum with_lg_page with_lg_hugepage -with_lg_page_sizes -with_version +enable_libdl enable_syscall enable_lazy_lock enable_zone_allocator @@ -849,6 +863,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE}' @@ -1101,6 +1116,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1238,7 +1262,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1391,6 +1415,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1424,8 +1449,13 @@ Optional Features: --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --disable-cxx Disable C++ integration --enable-autogen Automatically regenerate configure output + --enable-documentation Build documentation + --enable-shared Build shared libaries + --enable-static Build static libaries --enable-debug Build debugging code --disable-stats Disable statistics calculation/reporting + --enable-experimental-smallocx + Enable experimental smallocx API --enable-prof Enable allocation profiling --enable-prof-libunwind Use libunwind for backtracing --disable-prof-libgcc Do not use libgcc for backtracing @@ -1437,6 +1467,11 @@ Optional Features: Disable support for cache-oblivious allocation alignment --enable-log Support debug logging + --enable-readlinkat Use readlinkat over readlink + --enable-opt-safety-checks + Perform certain low-overhead checks, even in opt + mode + --disable-libdl Do not use libdl --disable-syscall Disable use of syscall(2) --enable-lazy-lock Enable lazy locking (only lock when multi-threaded) --disable-zone-allocator @@ -1450,6 +1485,8 @@ Optional Packages: --with-xslroot= XSL stylesheet root path --with-lg-vaddr= Number of significant virtual address bits + --with-version=..--g + Version string --with-rpath= Colon-separated rpath (ELF systems only) --with-mangling= Mangle symbols in --with-jemalloc-prefix= @@ -1470,10 +1507,6 @@ Optional Packages: Base 2 log of system page size --with-lg-hugepage= Base 2 log of system huge page size - --with-lg-page-sizes= - Base 2 logs of system page sizes to support - --with-version=..--g - Version string Some influential environment variables: CC C compiler command @@ -2699,6 +2732,9 @@ else fi +if test "x$XSLTPROC" = "xfalse" ; then + XSLROOT="" +fi CFLAGS=$CFLAGS @@ -3766,6 +3802,57 @@ fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Wextra" >&5 +$as_echo_n "checking whether compiler supports -Wextra... " >&6; } +T_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" +T_APPEND_V=-Wextra + if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${T_APPEND_V}" = "x" ; then + CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}${T_APPEND_V}" +else + CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS} ${T_APPEND_V}" +fi + + +if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${SPECIFIED_CFLAGS}" = "x" ; then + CFLAGS="${CONFIGURE_CFLAGS}${SPECIFIED_CFLAGS}" +else + CFLAGS="${CONFIGURE_CFLAGS} ${SPECIFIED_CFLAGS}" +fi + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_added=-Wextra + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_added= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CONFIGURE_CFLAGS="${T_CONFIGURE_CFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${SPECIFIED_CFLAGS}" = "x" ; then + CFLAGS="${CONFIGURE_CFLAGS}${SPECIFIED_CFLAGS}" +else + CFLAGS="${CONFIGURE_CFLAGS} ${SPECIFIED_CFLAGS}" +fi + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Wshorten-64-to-32" >&5 $as_echo_n "checking whether compiler supports -Wshorten-64-to-32... " >&6; } T_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" @@ -5933,6 +6020,69 @@ fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Wextra" >&5 +$as_echo_n "checking whether compiler supports -Wextra... " >&6; } +T_CONFIGURE_CXXFLAGS="${CONFIGURE_CXXFLAGS}" +T_APPEND_V=-Wextra + if test "x${CONFIGURE_CXXFLAGS}" = "x" -o "x${T_APPEND_V}" = "x" ; then + CONFIGURE_CXXFLAGS="${CONFIGURE_CXXFLAGS}${T_APPEND_V}" +else + CONFIGURE_CXXFLAGS="${CONFIGURE_CXXFLAGS} ${T_APPEND_V}" +fi + + +if test "x${CONFIGURE_CXXFLAGS}" = "x" -o "x${SPECIFIED_CXXFLAGS}" = "x" ; then + CXXFLAGS="${CONFIGURE_CXXFLAGS}${SPECIFIED_CXXFLAGS}" +else + CXXFLAGS="${CONFIGURE_CXXFLAGS} ${SPECIFIED_CXXFLAGS}" +fi + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + je_cv_cxxflags_added=-Wextra + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cxxflags_added= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CONFIGURE_CXXFLAGS="${T_CONFIGURE_CXXFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +if test "x${CONFIGURE_CXXFLAGS}" = "x" -o "x${SPECIFIED_CXXFLAGS}" = "x" ; then + CXXFLAGS="${CONFIGURE_CXXFLAGS}${SPECIFIED_CXXFLAGS}" +else + CXXFLAGS="${CONFIGURE_CXXFLAGS} ${SPECIFIED_CXXFLAGS}" +fi + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -g3" >&5 $as_echo_n "checking whether compiler supports -g3... " >&6; } T_CONFIGURE_CXXFLAGS="${CONFIGURE_CXXFLAGS}" @@ -7338,6 +7488,63 @@ fi done + + +# Check whether --with-version was given. +if test "${with_version+set}" = set; then : + withval=$with_version; + echo "${with_version}" | grep '^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$' 2>&1 1>/dev/null + if test $? -eq 0 ; then + echo "$with_version" > "${objroot}VERSION" + else + echo "${with_version}" | grep '^VERSION$' 2>&1 1>/dev/null + if test $? -ne 0 ; then + as_fn_error $? "${with_version} does not match ..--g or VERSION" "$LINENO" 5 + fi + fi + +else + + if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then + for pattern in '[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ + '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9][0-9]'; do + (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null + if test $? -eq 0 ; then + mv "${objroot}VERSION.tmp" "${objroot}VERSION" + break + fi + done + fi + rm -f "${objroot}VERSION.tmp" + +fi + + +if test ! -e "${objroot}VERSION" ; then + if test ! -e "${srcroot}VERSION" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Missing VERSION file, and unable to generate it; creating bogus VERSION" >&5 +$as_echo "Missing VERSION file, and unable to generate it; creating bogus VERSION" >&6; } + echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" + else + cp ${srcroot}VERSION ${objroot}VERSION + fi +fi +jemalloc_version=`cat "${objroot}VERSION"` +jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $1}'` +jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $2}'` +jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $3}'` +jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $4}'` +jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $5}'` + + + + + + + default_retain="0" maps_coalesce="1" DUMP_SYMS="${NM} -a" @@ -7527,6 +7734,9 @@ fi libprefix="" SOREV="${so}" PIC_CFLAGS="" + if test "${LG_SIZEOF_PTR}" = "3"; then + default_retain="1" + fi ;; *) { $as_echo "$as_me:${as_lineno-$LINENO}: result: Unsupported operating system: ${host}" >&5 @@ -8394,6 +8604,151 @@ if test "x${je_cv_format_printf}" = "xyes" ; then fi +SAVED_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Werror" >&5 +$as_echo_n "checking whether compiler supports -Werror... " >&6; } +T_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" +T_APPEND_V=-Werror + if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${T_APPEND_V}" = "x" ; then + CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}${T_APPEND_V}" +else + CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS} ${T_APPEND_V}" +fi + + +if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${SPECIFIED_CFLAGS}" = "x" ; then + CFLAGS="${CONFIGURE_CFLAGS}${SPECIFIED_CFLAGS}" +else + CFLAGS="${CONFIGURE_CFLAGS} ${SPECIFIED_CFLAGS}" +fi + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_added=-Werror + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_added= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CONFIGURE_CFLAGS="${T_CONFIGURE_CFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${SPECIFIED_CFLAGS}" = "x" ; then + CFLAGS="${CONFIGURE_CFLAGS}${SPECIFIED_CFLAGS}" +else + CFLAGS="${CONFIGURE_CFLAGS} ${SPECIFIED_CFLAGS}" +fi + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -herror_on_warning" >&5 +$as_echo_n "checking whether compiler supports -herror_on_warning... " >&6; } +T_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" +T_APPEND_V=-herror_on_warning + if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${T_APPEND_V}" = "x" ; then + CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}${T_APPEND_V}" +else + CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS} ${T_APPEND_V}" +fi + + +if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${SPECIFIED_CFLAGS}" = "x" ; then + CFLAGS="${CONFIGURE_CFLAGS}${SPECIFIED_CFLAGS}" +else + CFLAGS="${CONFIGURE_CFLAGS} ${SPECIFIED_CFLAGS}" +fi + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_added=-herror_on_warning + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_added= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CONFIGURE_CFLAGS="${T_CONFIGURE_CFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${SPECIFIED_CFLAGS}" = "x" ; then + CFLAGS="${CONFIGURE_CFLAGS}${SPECIFIED_CFLAGS}" +else + CFLAGS="${CONFIGURE_CFLAGS} ${SPECIFIED_CFLAGS}" +fi + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether format(printf, ...) attribute is compilable" >&5 +$as_echo_n "checking whether format(printf, ...) attribute is compilable... " >&6; } +if ${je_cv_format_arg+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +const char * __attribute__((__format_arg__(1))) foo(const char *format); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_format_arg=yes +else + je_cv_format_arg=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_format_arg" >&5 +$as_echo "$je_cv_format_arg" >&6; } + +CONFIGURE_CFLAGS="${SAVED_CONFIGURE_CFLAGS}" +if test "x${CONFIGURE_CFLAGS}" = "x" -o "x${SPECIFIED_CFLAGS}" = "x" ; then + CFLAGS="${CONFIGURE_CFLAGS}${SPECIFIED_CFLAGS}" +else + CFLAGS="${CONFIGURE_CFLAGS} ${SPECIFIED_CFLAGS}" +fi + + +if test "x${je_cv_format_arg}" = "xyes" ; then + $as_echo "#define JEMALLOC_HAVE_ATTR_FORMAT_ARG " >>confdefs.h + +fi + # Check whether --with-rpath was given. if test "${with_rpath+set}" = set; then : @@ -8692,6 +9047,55 @@ fi +# Check whether --enable-doc was given. +if test "${enable_doc+set}" = set; then : + enableval=$enable_doc; if test "x$enable_doc" = "xno" ; then + enable_doc="0" +else + enable_doc="1" +fi + +else + enable_doc="1" + +fi + + + +# Check whether --enable-shared was given. +if test "${enable_shared+set}" = set; then : + enableval=$enable_shared; if test "x$enable_shared" = "xno" ; then + enable_shared="0" +else + enable_shared="1" +fi + +else + enable_shared="1" + +fi + + + +# Check whether --enable-static was given. +if test "${enable_static+set}" = set; then : + enableval=$enable_static; if test "x$enable_static" = "xno" ; then + enable_static="0" +else + enable_static="1" +fi + +else + enable_static="1" + +fi + + + +if test "$enable_shared$enable_static" = "00" ; then + as_fn_error $? "Please enable one of shared or static builds" "$LINENO" 5 +fi + # Check whether --with-mangling was given. if test "${with_mangling+set}" = set; then : @@ -8742,7 +9146,7 @@ fi fi -public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" +public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" ac_fn_c_check_func "$LINENO" "memalign" "ac_cv_func_memalign" if test "x$ac_cv_func_memalign" = xyes; then : $as_echo "#define JEMALLOC_OVERRIDE_MEMALIGN " >>confdefs.h @@ -8907,7 +9311,6 @@ cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_unnamespace.sh" -cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/size_classes.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_rename.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_mangle.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc.sh" @@ -8920,7 +9323,6 @@ cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_symbols_jet.awk" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h" -cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h" @@ -8932,6 +9334,7 @@ cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.i cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:include/jemalloc/internal/jemalloc_internal_defs.h.in" cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:test/include/test/jemalloc_test_defs.h.in" + # Check whether --enable-debug was given. if test "${enable_debug+set}" = set; then : enableval=$enable_debug; if test "x$enable_debug" = "xno" ; then @@ -9374,6 +9777,25 @@ if test "x$enable_stats" = "x1" ; then fi +# Check whether --enable-experimental_smallocx was given. +if test "${enable_experimental_smallocx+set}" = set; then : + enableval=$enable_experimental_smallocx; if test "x$enable_experimental_smallocx" = "xno" ; then +enable_experimental_smallocx="0" +else +enable_experimental_smallocx="1" +fi + +else + enable_experimental_smallocx="0" + +fi + +if test "x$enable_experimental_smallocx" = "x1" ; then + $as_echo "#define JEMALLOC_EXPERIMENTAL_SMALLOCX_API 1" >>confdefs.h + +fi + + # Check whether --enable-prof was given. if test "${enable_prof+set}" = set; then : enableval=$enable_prof; if test "x$enable_prof" = "xno" ; then @@ -9856,6 +10278,43 @@ if test "x$enable_log" = "x1" ; then fi +# Check whether --enable-readlinkat was given. +if test "${enable_readlinkat+set}" = set; then : + enableval=$enable_readlinkat; if test "x$enable_readlinkat" = "xno" ; then + enable_readlinkat="0" +else + enable_readlinkat="1" +fi + +else + enable_readlinkat="0" + +fi + +if test "x$enable_readlinkat" = "x1" ; then + $as_echo "#define JEMALLOC_READLINKAT " >>confdefs.h + +fi + + +# Check whether --enable-opt-safety-checks was given. +if test "${enable_opt_safety_checks+set}" = set; then : + enableval=$enable_opt_safety_checks; if test "x$enable_opt_safety_checks" = "xno" ; then + enable_opt_safety_checks="0" +else + enable_opt_safety_checks="1" +fi + +else + enable_opt_safety_checks="0" + +fi + +if test "x$enable_opt_safety_checks" = "x1" ; then + $as_echo "#define JEMALLOC_OPT_SAFETY_CHECKS " >>confdefs.h + +fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a program using __builtin_unreachable is compilable" >&5 @@ -9996,6 +10455,50 @@ $as_echo "$je_cv_function_ffsl" >&6; } fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a program using __builtin_popcountl is compilable" >&5 +$as_echo_n "checking whether a program using __builtin_popcountl is compilable... " >&6; } +if ${je_cv_gcc_builtin_popcountl+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +#include + +int +main () +{ + + { + int rv = __builtin_popcountl(0x08); + printf("%d\n", rv); + } + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_gcc_builtin_popcountl=yes +else + je_cv_gcc_builtin_popcountl=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_gcc_builtin_popcountl" >&5 +$as_echo "$je_cv_gcc_builtin_popcountl" >&6; } + +if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then + $as_echo "#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount" >>confdefs.h + + $as_echo "#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl" >>confdefs.h + +fi + + # Check whether --with-lg_quantum was given. if test "${with_lg_quantum+set}" = set; then : withval=$with_lg_quantum; LG_QUANTA="$with_lg_quantum" @@ -10130,71 +10633,20 @@ cat >>confdefs.h <<_ACEOF _ACEOF - -# Check whether --with-lg_page_sizes was given. -if test "${with_lg_page_sizes+set}" = set; then : - withval=$with_lg_page_sizes; LG_PAGE_SIZES="$with_lg_page_sizes" +# Check whether --enable-libdl was given. +if test "${enable_libdl+set}" = set; then : + enableval=$enable_libdl; if test "x$enable_libdl" = "xno" ; then + enable_libdl="0" else - LG_PAGE_SIZES="$LG_PAGE" + enable_libdl="1" fi - - - -# Check whether --with-version was given. -if test "${with_version+set}" = set; then : - withval=$with_version; - echo "${with_version}" | grep '^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$' 2>&1 1>/dev/null - if test $? -eq 0 ; then - echo "$with_version" > "${objroot}VERSION" - else - echo "${with_version}" | grep '^VERSION$' 2>&1 1>/dev/null - if test $? -ne 0 ; then - as_fn_error $? "${with_version} does not match ..--g or VERSION" "$LINENO" 5 - fi - fi - else - - if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then - for pattern in '[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ - '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9][0-9]'; do - (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null - if test $? -eq 0 ; then - mv "${objroot}VERSION.tmp" "${objroot}VERSION" - break - fi - done - fi - rm -f "${objroot}VERSION.tmp" + enable_libdl="1" fi -if test ! -e "${objroot}VERSION" ; then - if test ! -e "${srcroot}VERSION" ; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: Missing VERSION file, and unable to generate it; creating bogus VERSION" >&5 -$as_echo "Missing VERSION file, and unable to generate it; creating bogus VERSION" >&6; } - echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" - else - cp ${srcroot}VERSION ${objroot}VERSION - fi -fi -jemalloc_version=`cat "${objroot}VERSION"` -jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $1}'` -jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $2}'` -jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $3}'` -jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $4}'` -jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $5}'` - - - - - - if test "x$abi" != "xpecoff" ; then @@ -10251,7 +10703,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthread_pthread_create" >&5 $as_echo "$ac_cv_lib_pthread_pthread_create" >&6; } if test "x$ac_cv_lib_pthread_pthread_create" = xyes; then : - T_APPEND_V=-lpthread + T_APPEND_V=-pthread if test "x${LIBS}" = "x" -o "x${T_APPEND_V}" = "x" ; then LIBS="${LIBS}${T_APPEND_V}" else @@ -10322,8 +10774,10 @@ fi wrap_syms="${wrap_syms} pthread_create" have_pthread="1" + + if test "x$enable_libdl" = "x1" ; then have_dlsym="1" - for ac_header in dlfcn.h + for ac_header in dlfcn.h do : ac_fn_c_check_header_mongrel "$LINENO" "dlfcn.h" "ac_cv_header_dlfcn_h" "$ac_includes_default" if test "x$ac_cv_header_dlfcn_h" = xyes; then : @@ -10384,11 +10838,15 @@ fi done - if test "x$have_dlsym" = "x1" ; then - $as_echo "#define JEMALLOC_HAVE_DLSYM " >>confdefs.h + if test "x$have_dlsym" = "x1" ; then + $as_echo "#define JEMALLOC_HAVE_DLSYM " >>confdefs.h + fi + else + have_dlsym="0" fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether pthread_atfork(3) is compilable" >&5 $as_echo_n "checking whether pthread_atfork(3) is compilable... " >&6; } if ${je_cv_pthread_atfork+:} false; then : @@ -11137,6 +11595,46 @@ $as_echo "$je_cv_gcc_atomic_atomics" >&6; } if test "x${je_cv_gcc_atomic_atomics}" = "xyes" ; then $as_echo "#define JEMALLOC_GCC_ATOMIC_ATOMICS 1" >>confdefs.h + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether GCC 8-bit __atomic atomics is compilable" >&5 +$as_echo_n "checking whether GCC 8-bit __atomic atomics is compilable... " >&6; } +if ${je_cv_gcc_u8_atomic_atomics+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + unsigned char x = 0; + int val = 1; + int y = __atomic_fetch_add(&x, val, __ATOMIC_RELAXED); + int after_add = (int)x; + return after_add == 1; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_gcc_u8_atomic_atomics=yes +else + je_cv_gcc_u8_atomic_atomics=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_gcc_u8_atomic_atomics" >&5 +$as_echo "$je_cv_gcc_u8_atomic_atomics" >&6; } + + if test "x${je_cv_gcc_u8_atomic_atomics}" = "xyes" ; then + $as_echo "#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1" >>confdefs.h + + fi fi @@ -11177,6 +11675,45 @@ $as_echo "$je_cv_gcc_sync_atomics" >&6; } if test "x${je_cv_gcc_sync_atomics}" = "xyes" ; then $as_echo "#define JEMALLOC_GCC_SYNC_ATOMICS 1" >>confdefs.h + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether GCC 8-bit __sync atomics is compilable" >&5 +$as_echo_n "checking whether GCC 8-bit __sync atomics is compilable... " >&6; } +if ${je_cv_gcc_u8_sync_atomics+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + unsigned char x = 0; + int before_add = __sync_fetch_and_add(&x, 1); + int after_add = (int)x; + return (before_add == 0) && (after_add == 1); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_gcc_u8_sync_atomics=yes +else + je_cv_gcc_u8_sync_atomics=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_gcc_u8_sync_atomics" >&5 +$as_echo "$je_cv_gcc_u8_sync_atomics" >&6; } + + if test "x${je_cv_gcc_u8_sync_atomics}" = "xyes" ; then + $as_echo "#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1" >>confdefs.h + + fi fi @@ -11433,102 +11970,6 @@ esac fi - - -if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to force 32-bit __sync_{add,sub}_and_fetch()" >&5 -$as_echo_n "checking whether to force 32-bit __sync_{add,sub}_and_fetch()... " >&6; } -if ${je_cv_sync_compare_and_swap_4+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - #include - -int -main () -{ - - #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 - { - uint32_t x32 = 0; - __sync_add_and_fetch(&x32, 42); - __sync_sub_and_fetch(&x32, 1); - } - #else - #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 is defined, no need to force - #endif - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - je_cv_sync_compare_and_swap_4=yes -else - je_cv_sync_compare_and_swap_4=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_sync_compare_and_swap_4" >&5 -$as_echo "$je_cv_sync_compare_and_swap_4" >&6; } - - if test "x${je_cv_sync_compare_and_swap_4}" = "xyes" ; then - $as_echo "#define JE_FORCE_SYNC_COMPARE_AND_SWAP_4 " >>confdefs.h - - fi - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to force 64-bit __sync_{add,sub}_and_fetch()" >&5 -$as_echo_n "checking whether to force 64-bit __sync_{add,sub}_and_fetch()... " >&6; } -if ${je_cv_sync_compare_and_swap_8+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - #include - -int -main () -{ - - #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 - { - uint64_t x64 = 0; - __sync_add_and_fetch(&x64, 42); - __sync_sub_and_fetch(&x64, 1); - } - #else - #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 is defined, no need to force - #endif - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - je_cv_sync_compare_and_swap_8=yes -else - je_cv_sync_compare_and_swap_8=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_sync_compare_and_swap_8" >&5 -$as_echo "$je_cv_sync_compare_and_swap_8" >&6; } - - if test "x${je_cv_sync_compare_and_swap_8}" = "xyes" ; then - $as_echo "#define JE_FORCE_SYNC_COMPARE_AND_SWAP_8 " >>confdefs.h - - fi - -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_clz" >&5 $as_echo_n "checking for __builtin_clz... " >&6; } if ${je_cv_builtin_clz+:} false; then : @@ -11616,47 +12057,6 @@ if test "x${je_cv_os_unfair_lock}" = "xyes" ; then fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Darwin OSSpin*() is compilable" >&5 -$as_echo_n "checking whether Darwin OSSpin*() is compilable... " >&6; } -if ${je_cv_osspin+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include -#include - -int -main () -{ - - OSSpinLock lock = 0; - OSSpinLockLock(&lock); - OSSpinLockUnlock(&lock); - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - je_cv_osspin=yes -else - je_cv_osspin=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_osspin" >&5 -$as_echo "$je_cv_osspin" >&6; } - -if test "x${je_cv_osspin}" = "xyes" ; then - $as_echo "#define JEMALLOC_OSSPIN " >>confdefs.h - -fi - - # Check whether --enable-zone-allocator was given. if test "${enable_zone_allocator+set}" = set; then : enableval=$enable_zone_allocator; if test "x$enable_zone_allocator" = "xno" ; then @@ -11708,9 +12108,7 @@ else fi -if test "x${have_pthread}" = "x1" -a "x${have_dlsym}" = "x1" \ - -a "x${je_cv_os_unfair_lock}" != "xyes" \ - -a "x${je_cv_osspin}" != "xyes" ; then +if test "x${have_pthread}" = "x1" -a "x${je_cv_os_unfair_lock}" != "xyes" ; then $as_echo "#define JEMALLOC_BACKGROUND_THREAD 1" >>confdefs.h fi @@ -12156,8 +12554,6 @@ ac_config_commands="$ac_config_commands include/jemalloc/internal/public_namespa ac_config_commands="$ac_config_commands include/jemalloc/internal/public_unnamespace.h" -ac_config_commands="$ac_config_commands include/jemalloc/internal/size_classes.h" - ac_config_commands="$ac_config_commands include/jemalloc/jemalloc_protos_jet.h" ac_config_commands="$ac_config_commands include/jemalloc/jemalloc_rename.h" @@ -12901,13 +13297,6 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 objroot="${objroot}" - SHELL="${SHELL}" - srcdir="${srcdir}" - objroot="${objroot}" - LG_QUANTA="${LG_QUANTA}" - LG_PAGE_SIZES="${LG_PAGE_SIZES}" - - srcdir="${srcdir}" objroot="${objroot}" @@ -12942,7 +13331,6 @@ do "include/jemalloc/internal/private_symbols_jet.awk") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/private_symbols_jet.awk" ;; "include/jemalloc/internal/public_namespace.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/public_namespace.h" ;; "include/jemalloc/internal/public_unnamespace.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/public_unnamespace.h" ;; - "include/jemalloc/internal/size_classes.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/size_classes.h" ;; "include/jemalloc/jemalloc_protos_jet.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc_protos_jet.h" ;; "include/jemalloc/jemalloc_rename.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc_rename.h" ;; "include/jemalloc/jemalloc_mangle.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc_mangle.h" ;; @@ -13544,10 +13932,6 @@ $as_echo "$as_me: executing $ac_file commands" >&6;} "include/jemalloc/internal/public_unnamespace.h":C) mkdir -p "${objroot}include/jemalloc/internal" "${srcdir}/include/jemalloc/internal/public_unnamespace.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/internal/public_unnamespace.h" - ;; - "include/jemalloc/internal/size_classes.h":C) - mkdir -p "${objroot}include/jemalloc/internal" - "${SHELL}" "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" 3 "${LG_PAGE_SIZES}" 2 > "${objroot}include/jemalloc/internal/size_classes.h" ;; "include/jemalloc/jemalloc_protos_jet.h":C) mkdir -p "${objroot}include/jemalloc" @@ -13688,12 +14072,20 @@ $as_echo " : ${JEMALLOC_PRIVATE_NAMESPACE}" >&6; } $as_echo "install_suffix : ${install_suffix}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: malloc_conf : ${config_malloc_conf}" >&5 $as_echo "malloc_conf : ${config_malloc_conf}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: documentation : ${enable_doc}" >&5 +$as_echo "documentation : ${enable_doc}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: shared libs : ${enable_shared}" >&5 +$as_echo "shared libs : ${enable_shared}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: static libs : ${enable_static}" >&5 +$as_echo "static libs : ${enable_static}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: autogen : ${enable_autogen}" >&5 $as_echo "autogen : ${enable_autogen}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: debug : ${enable_debug}" >&5 $as_echo "debug : ${enable_debug}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: stats : ${enable_stats}" >&5 $as_echo "stats : ${enable_stats}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: experimetal_smallocx : ${enable_experimental_smallocx}" >&5 +$as_echo "experimetal_smallocx : ${enable_experimental_smallocx}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: prof : ${enable_prof}" >&5 $as_echo "prof : ${enable_prof}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: prof-libunwind : ${enable_prof_libunwind}" >&5 diff --git a/deps/jemalloc/configure.ac b/deps/jemalloc/configure.ac index a6a08db08..f867172f7 100644 --- a/deps/jemalloc/configure.ac +++ b/deps/jemalloc/configure.ac @@ -175,6 +175,9 @@ fi ], XSLROOT="${DEFAULT_XSLROOT}" ) +if test "x$XSLTPROC" = "xfalse" ; then + XSLROOT="" +fi AC_SUBST([XSLROOT]) dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise, @@ -242,6 +245,7 @@ if test "x$GCC" = "xyes" ; then fi fi JE_CFLAGS_ADD([-Wall]) + JE_CFLAGS_ADD([-Wextra]) JE_CFLAGS_ADD([-Wshorten-64-to-32]) JE_CFLAGS_ADD([-Wsign-compare]) JE_CFLAGS_ADD([-Wundef]) @@ -289,6 +293,7 @@ if test "x$enable_cxx" = "x1" ; then AX_CXX_COMPILE_STDCXX([14], [noext], [optional]) if test "x${HAVE_CXX14}" = "x1" ; then JE_CXXFLAGS_ADD([-Wall]) + JE_CXXFLAGS_ADD([-Wextra]) JE_CXXFLAGS_ADD([-g3]) SAVED_LIBS="${LIBS}" @@ -512,7 +517,7 @@ CTARGET='-o $@' LDTARGET='-o $@' TEST_LD_MODE= EXTRA_LDFLAGS= -ARFLAGS='crus' +ARFLAGS='crs' AROUT=' $@' CC_MM=1 @@ -536,6 +541,66 @@ AC_PROG_NM AC_PROG_AWK +dnl ============================================================================ +dnl jemalloc version. +dnl + +AC_ARG_WITH([version], + [AS_HELP_STRING([--with-version=..--g], + [Version string])], + [ + echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null + if test $? -eq 0 ; then + echo "$with_version" > "${objroot}VERSION" + else + echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null + if test $? -ne 0 ; then + AC_MSG_ERROR([${with_version} does not match ..--g or VERSION]) + fi + fi + ], [ + dnl Set VERSION if source directory is inside a git repository. + if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then + dnl Pattern globs aren't powerful enough to match both single- and + dnl double-digit version numbers, so iterate over patterns to support up + dnl to version 99.99.99 without any accidental matches. + for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ + '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do + (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null + if test $? -eq 0 ; then + mv "${objroot}VERSION.tmp" "${objroot}VERSION" + break + fi + done + fi + rm -f "${objroot}VERSION.tmp" + ]) + +if test ! -e "${objroot}VERSION" ; then + if test ! -e "${srcroot}VERSION" ; then + AC_MSG_RESULT( + [Missing VERSION file, and unable to generate it; creating bogus VERSION]) + echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" + else + cp ${srcroot}VERSION ${objroot}VERSION + fi +fi +jemalloc_version=`cat "${objroot}VERSION"` +jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` +jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` +jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` +jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` +jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` +AC_SUBST([jemalloc_version]) +AC_SUBST([jemalloc_version_major]) +AC_SUBST([jemalloc_version_minor]) +AC_SUBST([jemalloc_version_bugfix]) +AC_SUBST([jemalloc_version_nrev]) +AC_SUBST([jemalloc_version_gid]) + dnl Platform-specific settings. abi and RPATH can probably be determined dnl programmatically, but doing so is error-prone, which makes it generally dnl not worth the trouble. @@ -673,6 +738,9 @@ case "${host}" in libprefix="" SOREV="${so}" PIC_CFLAGS="" + if test "${LG_SIZEOF_PTR}" = "3"; then + default_retain="1" + fi ;; *) AC_MSG_RESULT([Unsupported operating system: ${host}]) @@ -786,6 +854,18 @@ if test "x${je_cv_format_printf}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_PRINTF], [ ]) fi +dnl Check for format_arg(...) attribute support. +JE_CFLAGS_SAVE() +JE_CFLAGS_ADD([-Werror]) +JE_CFLAGS_ADD([-herror_on_warning]) +JE_COMPILABLE([format(printf, ...) attribute], [#include ], + [const char * __attribute__((__format_arg__(1))) foo(const char *format);], + [je_cv_format_arg]) +JE_CFLAGS_RESTORE() +if test "x${je_cv_format_arg}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_ARG], [ ]) +fi + dnl Support optional additions to rpath. AC_ARG_WITH([rpath], [AS_HELP_STRING([--with-rpath=], [Colon-separated rpath (ELF systems only)])], @@ -816,6 +896,49 @@ AC_PROG_RANLIB AC_PATH_PROG([LD], [ld], [false], [$PATH]) AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH]) +dnl Enable documentation +AC_ARG_ENABLE([doc], + [AS_HELP_STRING([--enable-documentation], [Build documentation])], +if test "x$enable_doc" = "xno" ; then + enable_doc="0" +else + enable_doc="1" +fi +, +enable_doc="1" +) +AC_SUBST([enable_doc]) + +dnl Enable shared libs +AC_ARG_ENABLE([shared], + [AS_HELP_STRING([--enable-shared], [Build shared libaries])], +if test "x$enable_shared" = "xno" ; then + enable_shared="0" +else + enable_shared="1" +fi +, +enable_shared="1" +) +AC_SUBST([enable_shared]) + +dnl Enable static libs +AC_ARG_ENABLE([static], + [AS_HELP_STRING([--enable-static], [Build static libaries])], +if test "x$enable_static" = "xno" ; then + enable_static="0" +else + enable_static="1" +fi +, +enable_static="1" +) +AC_SUBST([enable_static]) + +if test "$enable_shared$enable_static" = "00" ; then + AC_MSG_ERROR([Please enable one of shared or static builds]) +fi + dnl Perform no name mangling by default. AC_ARG_WITH([mangling], [AS_HELP_STRING([--with-mangling=], [Mangle symbols in ])], @@ -848,7 +971,7 @@ AC_ARG_WITH([export], fi] ) -public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" +public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" dnl Check for additional platform-specific public API functions. AC_CHECK_FUNC([memalign], [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ]) @@ -966,7 +1089,6 @@ cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_unnamespace.sh" -cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/size_classes.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_rename.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_mangle.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc.sh" @@ -979,7 +1101,6 @@ cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_symbols_jet.awk" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h" -cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h" @@ -991,6 +1112,10 @@ cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.i cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:include/jemalloc/internal/jemalloc_internal_defs.h.in" cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:test/include/test/jemalloc_test_defs.h.in" +dnl ============================================================================ +dnl jemalloc build options. +dnl + dnl Do not compile with debugging by default. AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], @@ -1043,6 +1168,22 @@ if test "x$enable_stats" = "x1" ; then fi AC_SUBST([enable_stats]) +dnl Do not enable smallocx by default. +AC_ARG_ENABLE([experimental_smallocx], + [AS_HELP_STRING([--enable-experimental-smallocx], [Enable experimental smallocx API])], +[if test "x$enable_experimental_smallocx" = "xno" ; then +enable_experimental_smallocx="0" +else +enable_experimental_smallocx="1" +fi +], +[enable_experimental_smallocx="0"] +) +if test "x$enable_experimental_smallocx" = "x1" ; then + AC_DEFINE([JEMALLOC_EXPERIMENTAL_SMALLOCX_API]) +fi +AC_SUBST([enable_experimental_smallocx]) + dnl Do not enable profiling by default. AC_ARG_ENABLE([prof], [AS_HELP_STRING([--enable-prof], [Enable allocation profiling])], @@ -1277,6 +1418,38 @@ if test "x$enable_log" = "x1" ; then fi AC_SUBST([enable_log]) +dnl Do not use readlinkat by default +AC_ARG_ENABLE([readlinkat], + [AS_HELP_STRING([--enable-readlinkat], [Use readlinkat over readlink])], +[if test "x$enable_readlinkat" = "xno" ; then + enable_readlinkat="0" +else + enable_readlinkat="1" +fi +], +[enable_readlinkat="0"] +) +if test "x$enable_readlinkat" = "x1" ; then + AC_DEFINE([JEMALLOC_READLINKAT], [ ]) +fi +AC_SUBST([enable_readlinkat]) + +dnl Avoid extra safety checks by default +AC_ARG_ENABLE([opt-safety-checks], + [AS_HELP_STRING([--enable-opt-safety-checks], + [Perform certain low-overhead checks, even in opt mode])], +[if test "x$enable_opt_safety_checks" = "xno" ; then + enable_opt_safety_checks="0" +else + enable_opt_safety_checks="1" +fi +], +[enable_opt_safety_checks="0"] +) +if test "x$enable_opt_safety_checks" = "x1" ; then + AC_DEFINE([JEMALLOC_OPT_SAFETY_CHECKS], [ ]) +fi +AC_SUBST([enable_opt_safety_checks]) JE_COMPILABLE([a program using __builtin_unreachable], [ void foo (void) { @@ -1333,6 +1506,21 @@ else fi fi +JE_COMPILABLE([a program using __builtin_popcountl], [ +#include +#include +#include +], [ + { + int rv = __builtin_popcountl(0x08); + printf("%d\n", rv); + } +], [je_cv_gcc_builtin_popcountl]) +if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNT], [__builtin_popcount]) + AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNTL], [__builtin_popcountl]) +fi + AC_ARG_WITH([lg_quantum], [AS_HELP_STRING([--with-lg-quantum=], [Base 2 log of minimum allocation alignment])], @@ -1430,70 +1618,20 @@ if test "x${LG_PAGE}" != "xundefined" -a \ fi AC_DEFINE_UNQUOTED([LG_HUGEPAGE], [${je_cv_lg_hugepage}]) -AC_ARG_WITH([lg_page_sizes], - [AS_HELP_STRING([--with-lg-page-sizes=], - [Base 2 logs of system page sizes to support])], - [LG_PAGE_SIZES="$with_lg_page_sizes"], [LG_PAGE_SIZES="$LG_PAGE"]) - dnl ============================================================================ -dnl jemalloc configuration. -dnl - -AC_ARG_WITH([version], - [AS_HELP_STRING([--with-version=..--g], - [Version string])], - [ - echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null - if test $? -eq 0 ; then - echo "$with_version" > "${objroot}VERSION" - else - echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null - if test $? -ne 0 ; then - AC_MSG_ERROR([${with_version} does not match ..--g or VERSION]) - fi - fi - ], [ - dnl Set VERSION if source directory is inside a git repository. - if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then - dnl Pattern globs aren't powerful enough to match both single- and - dnl double-digit version numbers, so iterate over patterns to support up - dnl to version 99.99.99 without any accidental matches. - for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ - '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do - (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null - if test $? -eq 0 ; then - mv "${objroot}VERSION.tmp" "${objroot}VERSION" - break - fi - done - fi - rm -f "${objroot}VERSION.tmp" - ]) - -if test ! -e "${objroot}VERSION" ; then - if test ! -e "${srcroot}VERSION" ; then - AC_MSG_RESULT( - [Missing VERSION file, and unable to generate it; creating bogus VERSION]) - echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" - else - cp ${srcroot}VERSION ${objroot}VERSION - fi +dnl Enable libdl by default. +AC_ARG_ENABLE([libdl], + [AS_HELP_STRING([--disable-libdl], + [Do not use libdl])], +[if test "x$enable_libdl" = "xno" ; then + enable_libdl="0" +else + enable_libdl="1" fi -jemalloc_version=`cat "${objroot}VERSION"` -jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` -jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` -jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` -jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` -jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` -AC_SUBST([jemalloc_version]) -AC_SUBST([jemalloc_version_major]) -AC_SUBST([jemalloc_version_minor]) -AC_SUBST([jemalloc_version_bugfix]) -AC_SUBST([jemalloc_version_nrev]) -AC_SUBST([jemalloc_version_gid]) +], +[enable_libdl="1"] +) +AC_SUBST([libdl]) dnl ============================================================================ dnl Configure pthreads. @@ -1503,20 +1641,26 @@ if test "x$abi" != "xpecoff" ; then AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) dnl Some systems may embed pthreads functionality in libc; check for libpthread dnl first, but try libc too before failing. - AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -lpthread)], + AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -pthread)], [AC_SEARCH_LIBS([pthread_create], , , AC_MSG_ERROR([libpthread is missing]))]) wrap_syms="${wrap_syms} pthread_create" have_pthread="1" - dnl Check if we have dlsym support. - have_dlsym="1" - AC_CHECK_HEADERS([dlfcn.h], - AC_CHECK_FUNC([dlsym], [], - [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]), - [have_dlsym="0"]) - if test "x$have_dlsym" = "x1" ; then - AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ]) + +dnl Check if we have dlsym support. + if test "x$enable_libdl" = "x1" ; then + have_dlsym="1" + AC_CHECK_HEADERS([dlfcn.h], + AC_CHECK_FUNC([dlsym], [], + [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]), + [have_dlsym="0"]) + if test "x$have_dlsym" = "x1" ; then + AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ]) + fi + else + have_dlsym="0" fi + JE_COMPILABLE([pthread_atfork(3)], [ #include ], [ @@ -1780,6 +1924,19 @@ JE_COMPILABLE([GCC __atomic atomics], [ ], [je_cv_gcc_atomic_atomics]) if test "x${je_cv_gcc_atomic_atomics}" = "xyes" ; then AC_DEFINE([JEMALLOC_GCC_ATOMIC_ATOMICS]) + + dnl check for 8-bit atomic support + JE_COMPILABLE([GCC 8-bit __atomic atomics], [ + ], [ + unsigned char x = 0; + int val = 1; + int y = __atomic_fetch_add(&x, val, __ATOMIC_RELAXED); + int after_add = (int)x; + return after_add == 1; + ], [je_cv_gcc_u8_atomic_atomics]) + if test "x${je_cv_gcc_u8_atomic_atomics}" = "xyes" ; then + AC_DEFINE([JEMALLOC_GCC_U8_ATOMIC_ATOMICS]) + fi fi dnl ============================================================================ @@ -1794,12 +1951,24 @@ JE_COMPILABLE([GCC __sync atomics], [ ], [je_cv_gcc_sync_atomics]) if test "x${je_cv_gcc_sync_atomics}" = "xyes" ; then AC_DEFINE([JEMALLOC_GCC_SYNC_ATOMICS]) + + dnl check for 8-bit atomic support + JE_COMPILABLE([GCC 8-bit __sync atomics], [ + ], [ + unsigned char x = 0; + int before_add = __sync_fetch_and_add(&x, 1); + int after_add = (int)x; + return (before_add == 0) && (after_add == 1); + ], [je_cv_gcc_u8_sync_atomics]) + if test "x${je_cv_gcc_u8_sync_atomics}" = "xyes" ; then + AC_DEFINE([JEMALLOC_GCC_U8_SYNC_ATOMICS]) + fi fi dnl ============================================================================ dnl Check for atomic(3) operations as provided on Darwin. dnl We need this not for the atomic operations (which are provided above), but -dnl rather for the OSSpinLock type it exposes. +dnl rather for the OS_unfair_lock type it exposes. JE_COMPILABLE([Darwin OSAtomic*()], [ #include @@ -1870,7 +2039,7 @@ if test "x${je_cv_madvise}" = "xyes" ; then if test "x${je_cv_madv_dontdump}" = "xyes" ; then AC_DEFINE([JEMALLOC_MADVISE_DONTDUMP], [ ]) fi - + dnl Check for madvise(..., MADV_[NO]HUGEPAGE). JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [ #include @@ -1889,40 +2058,6 @@ case "${host_cpu}" in esac fi -dnl ============================================================================ -dnl Check whether __sync_{add,sub}_and_fetch() are available despite -dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined. - -AC_DEFUN([JE_SYNC_COMPARE_AND_SWAP_CHECK],[ - AC_CACHE_CHECK([whether to force $1-bit __sync_{add,sub}_and_fetch()], - [je_cv_sync_compare_and_swap_$2], - [AC_LINK_IFELSE([AC_LANG_PROGRAM([ - #include - ], - [ - #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 - { - uint$1_t x$1 = 0; - __sync_add_and_fetch(&x$1, 42); - __sync_sub_and_fetch(&x$1, 1); - } - #else - #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 is defined, no need to force - #endif - ])], - [je_cv_sync_compare_and_swap_$2=yes], - [je_cv_sync_compare_and_swap_$2=no])]) - - if test "x${je_cv_sync_compare_and_swap_$2}" = "xyes" ; then - AC_DEFINE([JE_FORCE_SYNC_COMPARE_AND_SWAP_$2], [ ]) - fi -]) - -if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then - JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4) - JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8) -fi - dnl ============================================================================ dnl Check for __builtin_clz() and __builtin_clzl(). @@ -1965,21 +2100,6 @@ if test "x${je_cv_os_unfair_lock}" = "xyes" ; then AC_DEFINE([JEMALLOC_OS_UNFAIR_LOCK], [ ]) fi -dnl ============================================================================ -dnl Check for spinlock(3) operations as provided on Darwin. - -JE_COMPILABLE([Darwin OSSpin*()], [ -#include -#include -], [ - OSSpinLock lock = 0; - OSSpinLockLock(&lock); - OSSpinLockUnlock(&lock); -], [je_cv_osspin]) -if test "x${je_cv_osspin}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSSPIN], [ ]) -fi - dnl ============================================================================ dnl Darwin-related configuration. @@ -2032,9 +2152,7 @@ fi dnl ============================================================================ dnl Enable background threads if possible. -if test "x${have_pthread}" = "x1" -a "x${have_dlsym}" = "x1" \ - -a "x${je_cv_os_unfair_lock}" != "xyes" \ - -a "x${je_cv_osspin}" != "xyes" ; then +if test "x${have_pthread}" = "x1" -a "x${je_cv_os_unfair_lock}" != "xyes" ; then AC_DEFINE([JEMALLOC_BACKGROUND_THREAD]) fi @@ -2175,16 +2293,6 @@ AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [ srcdir="${srcdir}" objroot="${objroot}" ]) -AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ - mkdir -p "${objroot}include/jemalloc/internal" - "${SHELL}" "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" 3 "${LG_PAGE_SIZES}" 2 > "${objroot}include/jemalloc/internal/size_classes.h" -], [ - SHELL="${SHELL}" - srcdir="${srcdir}" - objroot="${objroot}" - LG_QUANTA="${LG_QUANTA}" - LG_PAGE_SIZES="${LG_PAGE_SIZES}" -]) AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [ mkdir -p "${objroot}include/jemalloc" cat "${srcdir}/include/jemalloc/jemalloc_protos.h.in" | sed -e 's/@je_@/jet_/g' > "${objroot}include/jemalloc/jemalloc_protos_jet.h" @@ -2277,9 +2385,13 @@ AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([malloc_conf : ${config_malloc_conf}]) +AC_MSG_RESULT([documentation : ${enable_doc}]) +AC_MSG_RESULT([shared libs : ${enable_shared}]) +AC_MSG_RESULT([static libs : ${enable_static}]) AC_MSG_RESULT([autogen : ${enable_autogen}]) AC_MSG_RESULT([debug : ${enable_debug}]) AC_MSG_RESULT([stats : ${enable_stats}]) +AC_MSG_RESULT([experimetal_smallocx : ${enable_experimental_smallocx}]) AC_MSG_RESULT([prof : ${enable_prof}]) AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) diff --git a/deps/jemalloc/doc/jemalloc.xml.in b/deps/jemalloc/doc/jemalloc.xml.in index 1e12fd3a8..7fecda7cb 100644 --- a/deps/jemalloc/doc/jemalloc.xml.in +++ b/deps/jemalloc/doc/jemalloc.xml.in @@ -424,7 +424,7 @@ for (i = 0; i < nbins; i++) { called repeatedly. General information that never changes during execution can be omitted by specifying g as a character within the opts string. Note that - malloc_message() uses the + malloc_stats_print() uses the mallctl*() functions internally, so inconsistent statistics can be reported if multiple threads use these functions simultaneously. If is specified during @@ -433,10 +433,11 @@ for (i = 0; i < nbins; i++) { arena statistics, respectively; b and l can be specified to omit per size class statistics for bins and large objects, respectively; x can be specified to omit all mutex - statistics. Unrecognized characters are silently ignored. Note that - thread caching may prevent some statistics from being completely up to - date, since extra locking would be required to merge counters that track - thread cache operations. + statistics; e can be used to omit extent statistics. + Unrecognized characters are silently ignored. Note that thread caching + may prevent some statistics from being completely up to date, since extra + locking would be required to merge counters that track thread cache + operations. The malloc_usable_size() function returns the usable size of the allocation pointed to by @@ -903,6 +904,23 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", + + + opt.confirm_conf + (bool) + r- + + Confirm-runtime-options-when-program-starts + enabled/disabled. If true, the string specified via + , the string pointed to by the + global variable malloc_conf, the name + of the file referenced by the symbolic link named + /etc/malloc.conf, and the value of + the environment variable MALLOC_CONF, will be printed in + order. Then, each option being set will be individually printed. This + option is disabled by default. + + opt.abort_conf @@ -943,16 +961,19 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", munmap 2 or equivalent (see stats.retained for related details). - This option is disabled by default unless discarding virtual memory is - known to trigger - platform-specific performance problems, e.g. for [64-bit] Linux, which - has a quirk in its virtual memory allocation algorithm that causes - semi-permanent VM map holes under normal jemalloc operation. Although - munmap - 2 causes issues on 32-bit Linux as - well, retaining virtual memory for 32-bit Linux is disabled by default - due to the practical possibility of address space exhaustion. - + It also makes jemalloc use + mmap2 + or equivalent in a more greedy way, mapping larger + chunks in one go. This option is disabled by default unless discarding + virtual memory is known to trigger platform-specific performance + problems, namely 1) for [64-bit] Linux, which has a quirk in its virtual + memory allocation algorithm that causes semi-permanent VM map holes + under normal jemalloc operation; and 2) for [64-bit] Windows, which + disallows split / merged regions with + MEM_RELEASE. Although the + same issues may present on 32-bit platforms as well, retaining virtual + memory for 32-bit Linux and Windows is disabled by default due to the + practical possibility of address space exhaustion. @@ -988,6 +1009,24 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", number of CPUs, or one if there is a single CPU. + + + opt.oversize_threshold + (size_t) + r- + + The threshold in bytes of which requests are considered + oversize. Allocation requests with greater sizes are fulfilled from a + dedicated arena (automatically managed, however not within + narenas), in order to reduce fragmentation by not + mixing huge allocations with small ones. In addition, the decay API + guarantees on the extents greater than the specified threshold may be + overridden. Note that requests with arena index specified via + MALLOCX_ARENA, or threads associated with explicit + arenas will not be considered. The default threshold is 8MiB. Values + not within large size classes disables this feature. + + opt.percpu_arena @@ -1009,7 +1048,7 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", opt.background_thread - (const bool) + (bool) r- Internal background worker threads enabled/disabled. @@ -1024,7 +1063,7 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", opt.max_background_threads - (const size_t) + (size_t) r- Maximum number of background threads that will be created @@ -1055,7 +1094,11 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", linkend="arena.i.dirty_decay_ms">arena.<i>.dirty_decay_ms for related dynamic control options. See opt.muzzy_decay_ms - for a description of muzzy pages. + for a description of muzzy pages.for a description of muzzy pages. Note + that when the oversize_threshold + feature is enabled, the arenas reserved for oversize requests may have + its own default decay settings. @@ -1763,10 +1806,11 @@ malloc_conf = "xmalloc:true";]]> to control allocation for arenas explicitly created via arenas.create such that all extents originate from an application-supplied extent allocator - (by specifying the custom extent hook functions during arena creation), - but the automatically created arenas will have already created extents - prior to the application having an opportunity to take over extent - allocation. + (by specifying the custom extent hook functions during arena creation). + However, the API guarantees for the automatically created arenas may be + relaxed -- hooks set there may be called in a "best effort" fashion; in + addition there may be extents created prior to the application having an + opportunity to take over extent allocation. + + + stats.arenas.<i>.extent_avail + (size_t) + r- + [] + + Number of allocated (but unused) extent structs in this + arena. + + stats.arenas.<i>.base @@ -2760,6 +2815,28 @@ struct extent_hooks_s { all bin size classes. + + + stats.arenas.<i>.small.nfills + (uint64_t) + r- + [] + + Cumulative number of tcache fills by all small size + classes. + + + + + stats.arenas.<i>.small.nflushes + (uint64_t) + r- + [] + + Cumulative number of tcache flushes by all small size + classes. + + stats.arenas.<i>.large.allocated @@ -2810,6 +2887,28 @@ struct extent_hooks_s { all large size classes. + + + stats.arenas.<i>.large.nfills + (uint64_t) + r- + [] + + Cumulative number of tcache fills by all large size + classes. + + + + + stats.arenas.<i>.large.nflushes + (uint64_t) + r- + [] + + Cumulative number of tcache flushes by all large size + classes. + + stats.arenas.<i>.bins.<j>.nmalloc @@ -2909,6 +3008,17 @@ struct extent_hooks_s { Current number of slabs. + + + + stats.arenas.<i>.bins.<j>.nonfull_slabs + (size_t) + r- + [] + + Current number of nonfull slabs. + + stats.arenas.<i>.bins.<j>.mutex.{counter} @@ -2922,6 +3032,30 @@ struct extent_hooks_s { counters. + + + stats.arenas.<i>.extents.<j>.n{extent_type} + (size_t) + r- + [] + + Number of extents of the given type in this arena in + the bucket corresponding to page size index <j>. The extent type + is one of dirty, muzzy, or retained. + + + + + stats.arenas.<i>.extents.<j>.{extent_type}_bytes + (size_t) + r- + [] + + Sum of the bytes managed by extents of the given type + in this arena in the bucket corresponding to page size index <j>. + The extent type is one of dirty, muzzy, or retained. + + stats.arenas.<i>.lextents.<j>.nmalloc diff --git a/deps/jemalloc/include/jemalloc/internal/arena_externs.h b/deps/jemalloc/include/jemalloc/internal/arena_externs.h index 4b3732b41..a4523ae0c 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_externs.h @@ -3,8 +3,8 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/extent_dss.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" extern ssize_t opt_dirty_decay_ms; @@ -16,13 +16,17 @@ extern const char *percpu_arena_mode_names[]; extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS]; extern malloc_mutex_t arenas_lock; +extern size_t opt_oversize_threshold; +extern size_t oversize_threshold; + void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_t *bstats, arena_stats_large_t *lstats); + bin_stats_t *bstats, arena_stats_large_t *lstats, + arena_stats_extents_t *estats); void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); #ifdef JEMALLOC_JET @@ -56,16 +60,17 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero); void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero, tcache_t *tcache); -void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize); +void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize); void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); -void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, - extent_t *extent, void *ptr); +void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *extent, void *ptr); void arena_dalloc_small(tsdn_t *tsdn, void *ptr); bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); + size_t extra, bool zero, size_t *newsize); void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache); + size_t size, size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); ssize_t arena_dirty_decay_ms_default_get(void); @@ -79,7 +84,12 @@ void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -void arena_boot(void); +bool arena_init_huge(void); +bool arena_is_huge(unsigned arena_ind); +arena_t *arena_choose_huge(tsd_t *tsd); +bin_t *arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard); +void arena_boot(sc_data_t *sc_data); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h b/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h index 2b7e77e72..dd926575f 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h @@ -4,10 +4,36 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" +JEMALLOC_ALWAYS_INLINE bool +arena_has_default_hooks(arena_t *arena) { + return (extent_hooks_get(arena) == &extent_hooks_default); +} + +JEMALLOC_ALWAYS_INLINE arena_t * +arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { + if (arena != NULL) { + return arena; + } + + /* + * For huge allocations, use the dedicated huge arena if both are true: + * 1) is using auto arena selection (i.e. arena == NULL), and 2) the + * thread is not assigned to a manual arena. + */ + if (unlikely(size >= oversize_threshold)) { + arena_t *tsd_arena = tsd_arena_get(tsd); + if (tsd_arena == NULL || arena_is_auto(tsd_arena)) { + return arena_choose_huge(tsd); + } + } + + return arena_choose(tsd, NULL); +} + JEMALLOC_ALWAYS_INLINE prof_tctx_t * arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { cassert(config_prof); @@ -28,7 +54,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { } JEMALLOC_ALWAYS_INLINE void -arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize, +arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -47,7 +73,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize, } static inline void -arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) { +arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -57,6 +83,32 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) { large_prof_tctx_reset(tsdn, extent); } +JEMALLOC_ALWAYS_INLINE nstime_t +arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, + alloc_ctx_t *alloc_ctx) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + /* + * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're + * sure we have a sampled allocation. + */ + assert(!extent_slab_get(extent)); + return large_prof_alloc_time_get(extent); +} + +JEMALLOC_ALWAYS_INLINE void +arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, + nstime_t t) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + assert(!extent_slab_get(extent)); + large_prof_alloc_time_set(extent, t); +} + JEMALLOC_ALWAYS_INLINE void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { tsd_t *tsd; @@ -83,14 +135,33 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { arena_decay_ticks(tsdn, arena, 1); } +/* Purge a single extent to retained / unmapped directly. */ +JEMALLOC_ALWAYS_INLINE void +arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks, + extent_t *extent) { + size_t extent_size = extent_size_get(extent); + extent_dalloc_wrapper(tsdn, arena, + r_extent_hooks, extent); + if (config_stats) { + /* Update stats accordingly. */ + arena_stats_lock(tsdn, &arena->stats); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->nmadvise, 1); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE); + arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, + extent_size); + arena_stats_unlock(tsdn, &arena->stats); + } +} + JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); - assert(size != 0); if (likely(tcache != NULL)) { - if (likely(size <= SMALL_MAXCLASS)) { + if (likely(size <= SC_SMALL_MAXCLASS)) { return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache, size, ind, zero, slow_path); } @@ -119,7 +190,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) { szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } @@ -152,11 +223,21 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { /* Only slab members should be looked up via interior pointers. */ assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } +static inline void +arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) { + if (config_prof && unlikely(szind < SC_NBINS)) { + arena_dalloc_promoted(tsdn, ptr, NULL, true); + } else { + extent_t *extent = iealloc(tsdn, ptr); + large_dalloc(tsdn, extent); + } +} + static inline void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { assert(ptr != NULL); @@ -173,13 +254,28 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } if (likely(slab)) { /* Small allocation. */ arena_dalloc_small(tsdn, ptr); + } else { + arena_dalloc_large_no_tcache(tsdn, ptr, szind); + } +} + +JEMALLOC_ALWAYS_INLINE void +arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind, + bool slow_path) { + if (szind < nhbins) { + if (config_prof && unlikely(szind < SC_NBINS)) { + arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); + } else { + tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind, + slow_path); + } } else { extent_t *extent = iealloc(tsdn, ptr); large_dalloc(tsdn, extent); @@ -203,7 +299,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, if (alloc_ctx != NULL) { szind = alloc_ctx->szind; slab = alloc_ctx->slab; - assert(szind != NSIZES); + assert(szind != SC_NSIZES); } else { rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, @@ -215,7 +311,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } @@ -224,25 +320,14 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, slow_path); } else { - if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { - arena_dalloc_promoted(tsdn, ptr, tcache, - slow_path); - } else { - tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, - szind, slow_path); - } - } else { - extent_t *extent = iealloc(tsdn, ptr); - large_dalloc(tsdn, extent); - } + arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); } } static inline void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); szind_t szind; bool slab; @@ -252,7 +337,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if ((config_prof && opt_prof) || config_debug) { @@ -264,7 +349,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { (uintptr_t)ptr, true, &szind, &slab); assert(szind == sz_size2index(size)); - assert((config_prof && opt_prof) || slab == (szind < NBINS)); + assert((config_prof && opt_prof) || slab == (szind < SC_NBINS)); if (config_debug) { extent_t *extent = rtree_extent_read(tsdn, @@ -278,8 +363,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { /* Small allocation. */ arena_dalloc_small(tsdn, ptr); } else { - extent_t *extent = iealloc(tsdn, ptr); - large_dalloc(tsdn, extent); + arena_dalloc_large_no_tcache(tsdn, ptr, szind); } } @@ -288,7 +372,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, alloc_ctx_t *alloc_ctx, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); if (unlikely(tcache == NULL)) { arena_sdalloc_no_tcache(tsdn, ptr, size); @@ -297,7 +381,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, szind_t szind; bool slab; - UNUSED alloc_ctx_t local_ctx; + alloc_ctx_t local_ctx; if (config_prof && opt_prof) { if (alloc_ctx == NULL) { /* Uncommon case and should be a static check. */ @@ -318,7 +402,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if (config_debug) { @@ -336,18 +420,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, slow_path); } else { - if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { - arena_dalloc_promoted(tsdn, ptr, tcache, - slow_path); - } else { - tcache_dalloc_large(tsdn_tsd(tsdn), - tcache, ptr, szind, slow_path); - } - } else { - extent_t *extent = iealloc(tsdn, ptr); - large_dalloc(tsdn, extent); - } + arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); } } diff --git a/deps/jemalloc/include/jemalloc/internal/arena_stats.h b/deps/jemalloc/include/jemalloc/internal/arena_stats.h index 5f3dca8b1..23949ed92 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_stats.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_stats.h @@ -4,7 +4,9 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex_prof.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" + +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS /* * In those architectures that support 64-bit atomics, we use atomic updates for @@ -33,6 +35,13 @@ struct arena_stats_large_s { * periodically merges into this counter. */ arena_stats_u64_t nrequests; /* Partially derived. */ + /* + * Number of tcache fills / flushes for large (similarly, periodically + * merged). Note that there is no large tcache batch-fill currently + * (i.e. only fill 1 at a time); however flush may be batched. + */ + arena_stats_u64_t nfills; /* Partially derived. */ + arena_stats_u64_t nflushes; /* Partially derived. */ /* Current number of allocations of this size class. */ size_t curlextents; /* Derived. */ @@ -48,6 +57,22 @@ struct arena_stats_decay_s { arena_stats_u64_t purged; }; +typedef struct arena_stats_extents_s arena_stats_extents_t; +struct arena_stats_extents_s { + /* + * Stats for a given index in the range [0, SC_NPSIZES] in an extents_t. + * We track both bytes and # of extents: two extents in the same bucket + * may have different sizes if adjacent size classes differ by more than + * a page, so bytes cannot always be derived from # of extents. + */ + atomic_zu_t ndirty; + atomic_zu_t dirty_bytes; + atomic_zu_t nmuzzy; + atomic_zu_t muzzy_bytes; + atomic_zu_t nretained; + atomic_zu_t retained_bytes; +}; + /* * Arena stats. Note that fields marked "derived" are not directly maintained * within the arena code; rather their values are derived during stats merge @@ -69,6 +94,9 @@ struct arena_stats_s { */ atomic_zu_t retained; /* Derived. */ + /* Number of extent_t structs allocated by base, but not being used. */ + atomic_zu_t extent_avail; + arena_stats_decay_t decay_dirty; arena_stats_decay_t decay_muzzy; @@ -80,22 +108,27 @@ struct arena_stats_s { atomic_zu_t allocated_large; /* Derived. */ arena_stats_u64_t nmalloc_large; /* Derived. */ arena_stats_u64_t ndalloc_large; /* Derived. */ + arena_stats_u64_t nfills_large; /* Derived. */ + arena_stats_u64_t nflushes_large; /* Derived. */ arena_stats_u64_t nrequests_large; /* Derived. */ + /* VM space had to be leaked (undocumented). Normally 0. */ + atomic_zu_t abandoned_vm; + /* Number of bytes cached in tcache associated with this arena. */ atomic_zu_t tcache_bytes; /* Derived. */ mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]; /* One element for each large size class. */ - arena_stats_large_t lstats[NSIZES - NBINS]; + arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; /* Arena uptime. */ nstime_t uptime; }; static inline bool -arena_stats_init(UNUSED tsdn_t *tsdn, arena_stats_t *arena_stats) { +arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { if (config_debug) { for (size_t i = 0; i < sizeof(arena_stats_t); i++) { assert(((char *)arena_stats)[i] == 0); @@ -147,11 +180,11 @@ arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, #endif } -UNUSED static inline void +static inline void arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, arena_stats_u64_t *p, uint64_t x) { #ifdef JEMALLOC_ATOMIC_U64 - UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); + uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); assert(r - x <= r); #else malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); @@ -176,7 +209,8 @@ arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) { } static inline size_t -arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) { +arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p) { #ifdef JEMALLOC_ATOMIC_U64 return atomic_load_zu(p, ATOMIC_RELAXED); #else @@ -186,8 +220,8 @@ arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) { } static inline void -arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, - size_t x) { +arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p, size_t x) { #ifdef JEMALLOC_ATOMIC_U64 atomic_fetch_add_zu(p, x, ATOMIC_RELAXED); #else @@ -198,10 +232,10 @@ arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, } static inline void -arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, - size_t x) { +arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p, size_t x) { #ifdef JEMALLOC_ATOMIC_U64 - UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); + size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); assert(r - x <= r); #else malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); @@ -218,11 +252,12 @@ arena_stats_accum_zu(atomic_zu_t *dst, size_t src) { } static inline void -arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, +arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, szind_t szind, uint64_t nrequests) { arena_stats_lock(tsdn, arena_stats); - arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind - - NBINS].nrequests, nrequests); + arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS]; + arena_stats_add_u64(tsdn, arena_stats, &lstats->nrequests, nrequests); + arena_stats_add_u64(tsdn, arena_stats, &lstats->nflushes, 1); arena_stats_unlock(tsdn, arena_stats); } @@ -233,5 +268,4 @@ arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) { arena_stats_unlock(tsdn, arena_stats); } - #endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h b/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h index 38bc95962..eeab57fd6 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h @@ -10,7 +10,7 @@ #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/ticker.h" @@ -90,6 +90,9 @@ struct arena_s { */ atomic_u_t nthreads[2]; + /* Next bin shard for binding new threads. Synchronization: atomic. */ + atomic_u_t binshard_next; + /* * When percpu_arena is enabled, to amortize the cost of reading / * updating the current CPU id, track the most recent thread accessing @@ -113,7 +116,6 @@ struct arena_s { /* Synchronization: internal. */ prof_accum_t prof_accum; - uint64_t prof_accumbytes; /* * PRNG state for cache index randomization of large allocation base @@ -196,6 +198,7 @@ struct arena_s { * Synchronization: extent_avail_mtx. */ extent_tree_t extent_avail; + atomic_zu_t extent_avail_cnt; malloc_mutex_t extent_avail_mtx; /* @@ -203,7 +206,7 @@ struct arena_s { * * Synchronization: internal. */ - bin_t bins[NBINS]; + bins_t bins[SC_NBINS]; /* * Base allocator, from which arena metadata are allocated. diff --git a/deps/jemalloc/include/jemalloc/internal/arena_types.h b/deps/jemalloc/include/jemalloc/internal/arena_types.h index 70001b5f1..624937e4f 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_types.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_types.h @@ -1,13 +1,15 @@ #ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H #define JEMALLOC_INTERNAL_ARENA_TYPES_H +#include "jemalloc/internal/sc.h" + /* Maximum number of regions in one slab. */ -#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN) +#define LG_SLAB_MAXREGS (LG_PAGE - SC_LG_TINY_MIN) #define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) /* Default decay times in milliseconds. */ #define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) -#define MUZZY_DECAY_MS_DEFAULT ZD(10 * 1000) +#define MUZZY_DECAY_MS_DEFAULT (0) /* Number of event ticks between time checks. */ #define DECAY_NTICKS_PER_UPDATE 1000 @@ -40,4 +42,10 @@ typedef enum { #define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base) #define PERCPU_ARENA_DEFAULT percpu_arena_disabled +/* + * When allocation_size >= oversize_threshold, use the dedicated huge arena + * (unless have explicitly spicified arena index). 0 disables the feature. + */ +#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20) + #endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/atomic.h b/deps/jemalloc/include/jemalloc/internal/atomic.h index adadb1a3a..a76f54cee 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic.h @@ -1,12 +1,19 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_H #define JEMALLOC_INTERNAL_ATOMIC_H -#define ATOMIC_INLINE static inline +#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE +#define JEMALLOC_U8_ATOMICS #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) # include "jemalloc/internal/atomic_gcc_atomic.h" +# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(JEMALLOC_GCC_SYNC_ATOMICS) # include "jemalloc/internal/atomic_gcc_sync.h" +# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(_MSC_VER) # include "jemalloc/internal/atomic_msvc.h" #elif defined(JEMALLOC_C11_ATOMICS) @@ -66,6 +73,8 @@ JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR) JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR) +JEMALLOC_GENERATE_INT_ATOMICS(uint8_t, u8, 0) + JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2) #ifdef JEMALLOC_ATOMIC_U64 diff --git a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h index 6b73a14f8..471515e82 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h @@ -67,7 +67,8 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ + UNUSED type *expected, type desired, \ + atomic_memory_order_t success_mo, \ atomic_memory_order_t failure_mo) { \ return __atomic_compare_exchange(&a->repr, expected, &desired, \ true, atomic_enum_to_builtin(success_mo), \ @@ -76,7 +77,8 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ + UNUSED type *expected, type desired, \ + atomic_memory_order_t success_mo, \ atomic_memory_order_t failure_mo) { \ return __atomic_compare_exchange(&a->repr, expected, &desired, \ false, \ diff --git a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h index 30846e4d2..e02b7cbe3 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h @@ -27,8 +27,10 @@ atomic_fence(atomic_memory_order_t mo) { asm volatile("" ::: "memory"); # if defined(__i386__) || defined(__x86_64__) /* This is implicit on x86. */ -# elif defined(__ppc__) +# elif defined(__ppc64__) asm volatile("lwsync"); +# elif defined(__ppc__) + asm volatile("sync"); # elif defined(__sparc__) && defined(__arch64__) if (mo == atomic_memory_order_acquire) { asm volatile("membar #LoadLoad | #LoadStore"); @@ -113,8 +115,8 @@ atomic_store_##short_type(atomic_##short_type##_t *a, \ } \ \ ATOMIC_INLINE type \ -atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ +atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ + atomic_memory_order_t mo) { \ /* \ * Because of FreeBSD, we care about gcc 4.2, which doesn't have\ * an atomic exchange builtin. We fake it with a CAS loop. \ @@ -129,8 +131,9 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ @@ -142,8 +145,9 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ } \ ATOMIC_INLINE bool \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ diff --git a/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h b/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h index 3209aa49f..0f997e18b 100644 --- a/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h @@ -8,7 +8,6 @@ extern atomic_b_t background_thread_enabled_state; extern size_t n_background_threads; extern size_t max_background_threads; extern background_thread_info_t *background_thread_info; -extern bool can_enable_background_thread; bool background_thread_create(tsd_t *tsd, unsigned arena_ind); bool background_threads_enable(tsd_t *tsd); diff --git a/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h b/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h index ef50231e8..f85e86fa3 100644 --- a/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h +++ b/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h @@ -15,7 +15,12 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) { JEMALLOC_ALWAYS_INLINE background_thread_info_t * arena_background_thread_info_get(arena_t *arena) { unsigned arena_ind = arena_ind_get(arena); - return &background_thread_info[arena_ind % ncpus]; + return &background_thread_info[arena_ind % max_background_threads]; +} + +JEMALLOC_ALWAYS_INLINE background_thread_info_t * +background_thread_info_get(size_t ind) { + return &background_thread_info[ind % max_background_threads]; } JEMALLOC_ALWAYS_INLINE uint64_t diff --git a/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h b/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h index c1107dfe9..c02aa434c 100644 --- a/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h @@ -9,6 +9,7 @@ #define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX #define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT +#define DEFAULT_NUM_BACKGROUND_THREAD 4 typedef enum { background_thread_stopped, diff --git a/deps/jemalloc/include/jemalloc/internal/base_structs.h b/deps/jemalloc/include/jemalloc/internal/base_structs.h index 2102247ac..07f214eb2 100644 --- a/deps/jemalloc/include/jemalloc/internal/base_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/base_structs.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" /* Embedded at the beginning of every block of base-managed virtual memory. */ struct base_block_s { @@ -46,7 +46,7 @@ struct base_s { base_block_t *blocks; /* Heap of extents that track unused trailing space within blocks. */ - extent_heap_t avail[NSIZES]; + extent_heap_t avail[SC_NSIZES]; /* Stats, only maintained if config_stats. */ size_t allocated; diff --git a/deps/jemalloc/include/jemalloc/internal/bin.h b/deps/jemalloc/include/jemalloc/internal/bin.h index 9b416ada7..8547e8930 100644 --- a/deps/jemalloc/include/jemalloc/internal/bin.h +++ b/deps/jemalloc/include/jemalloc/internal/bin.h @@ -1,10 +1,12 @@ #ifndef JEMALLOC_INTERNAL_BIN_H #define JEMALLOC_INTERNAL_BIN_H +#include "jemalloc/internal/bin_stats.h" +#include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/extent_types.h" #include "jemalloc/internal/extent_structs.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/bin_stats.h" +#include "jemalloc/internal/sc.h" /* * A bin contains a set of extents that are currently being used for slab @@ -41,6 +43,9 @@ struct bin_info_s { /* Total number of regions in a slab for this bin's size class. */ uint32_t nregs; + /* Number of sharded bins in each arena for this size class. */ + uint32_t n_shards; + /* * Metadata used to manipulate bitmaps for slabs associated with this * bin. @@ -48,8 +53,7 @@ struct bin_info_s { bitmap_info_t bitmap_info; }; -extern const bin_info_t bin_infos[NBINS]; - +extern bin_info_t bin_infos[SC_NBINS]; typedef struct bin_s bin_t; struct bin_s { @@ -78,6 +82,18 @@ struct bin_s { bin_stats_t stats; }; +/* A set of sharded bins of the same size class. */ +typedef struct bins_s bins_t; +struct bins_s { + /* Sharded bins. Dynamically sized. */ + bin_t *bin_shards; +}; + +void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]); +bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards); +void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]); + /* Initializes a bin to empty. Returns true on error. */ bool bin_init(bin_t *bin); @@ -90,7 +106,7 @@ void bin_postfork_child(tsdn_t *tsdn, bin_t *bin); static inline void bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { malloc_mutex_lock(tsdn, &bin->lock); - malloc_mutex_prof_read(tsdn, &dst_bin_stats->mutex_data, &bin->lock); + malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock); dst_bin_stats->nmalloc += bin->stats.nmalloc; dst_bin_stats->ndalloc += bin->stats.ndalloc; dst_bin_stats->nrequests += bin->stats.nrequests; @@ -100,6 +116,7 @@ bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { dst_bin_stats->nslabs += bin->stats.nslabs; dst_bin_stats->reslabs += bin->stats.reslabs; dst_bin_stats->curslabs += bin->stats.curslabs; + dst_bin_stats->nonfull_slabs += bin->stats.nonfull_slabs; malloc_mutex_unlock(tsdn, &bin->lock); } diff --git a/deps/jemalloc/include/jemalloc/internal/bin_stats.h b/deps/jemalloc/include/jemalloc/internal/bin_stats.h index 86e673ec4..d04519c82 100644 --- a/deps/jemalloc/include/jemalloc/internal/bin_stats.h +++ b/deps/jemalloc/include/jemalloc/internal/bin_stats.h @@ -45,6 +45,9 @@ struct bin_stats_s { /* Current number of slabs in this bin. */ size_t curslabs; + /* Current size of nonfull slabs heap in this bin. */ + size_t nonfull_slabs; + mutex_prof_data_t mutex_data; }; diff --git a/deps/jemalloc/include/jemalloc/internal/bin_types.h b/deps/jemalloc/include/jemalloc/internal/bin_types.h new file mode 100644 index 000000000..3533606b9 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/bin_types.h @@ -0,0 +1,17 @@ +#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H +#define JEMALLOC_INTERNAL_BIN_TYPES_H + +#include "jemalloc/internal/sc.h" + +#define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH) +#define N_BIN_SHARDS_DEFAULT 1 + +/* Used in TSD static initializer only. Real init in arena_bind(). */ +#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}} + +typedef struct tsd_binshards_s tsd_binshards_t; +struct tsd_binshards_s { + uint8_t binshard[SC_NBINS]; +}; + +#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/bit_util.h b/deps/jemalloc/include/jemalloc/internal/bit_util.h index 8d078a8a3..c045eb868 100644 --- a/deps/jemalloc/include/jemalloc/internal/bit_util.h +++ b/deps/jemalloc/include/jemalloc/internal/bit_util.h @@ -27,6 +27,25 @@ ffs_u(unsigned bitmap) { return JEMALLOC_INTERNAL_FFS(bitmap); } +#ifdef JEMALLOC_INTERNAL_POPCOUNTL +BIT_UTIL_INLINE unsigned +popcount_lu(unsigned long bitmap) { + return JEMALLOC_INTERNAL_POPCOUNTL(bitmap); +} +#endif + +/* + * Clears first unset bit in bitmap, and returns + * place of bit. bitmap *must not* be 0. + */ + +BIT_UTIL_INLINE size_t +cfs_lu(unsigned long* bitmap) { + size_t bit = ffs_lu(*bitmap) - 1; + *bitmap ^= ZU(1) << bit; + return bit; +} + BIT_UTIL_INLINE unsigned ffs_zu(size_t bitmap) { #if LG_SIZEOF_PTR == LG_SIZEOF_INT @@ -63,6 +82,22 @@ ffs_u32(uint32_t bitmap) { BIT_UTIL_INLINE uint64_t pow2_ceil_u64(uint64_t x) { +#if (defined(__amd64__) || defined(__x86_64__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + if(unlikely(x <= 1)) { + return x; + } + size_t msb_on_index; +#if (defined(__amd64__) || defined(__x86_64__)) + asm ("bsrq %1, %0" + : "=r"(msb_on_index) // Outputs. + : "r"(x-1) // Inputs. + ); +#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + msb_on_index = (63 ^ __builtin_clzll(x - 1)); +#endif + assert(msb_on_index < 63); + return 1ULL << (msb_on_index + 1); +#else x--; x |= x >> 1; x |= x >> 2; @@ -72,10 +107,27 @@ pow2_ceil_u64(uint64_t x) { x |= x >> 32; x++; return x; +#endif } BIT_UTIL_INLINE uint32_t pow2_ceil_u32(uint32_t x) { +#if ((defined(__i386__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) && (!defined(__s390__))) + if(unlikely(x <= 1)) { + return x; + } + size_t msb_on_index; +#if (defined(__i386__)) + asm ("bsr %1, %0" + : "=r"(msb_on_index) // Outputs. + : "r"(x-1) // Inputs. + ); +#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + msb_on_index = (31 ^ __builtin_clz(x - 1)); +#endif + assert(msb_on_index < 31); + return 1U << (msb_on_index + 1); +#else x--; x |= x >> 1; x |= x >> 2; @@ -84,6 +136,7 @@ pow2_ceil_u32(uint32_t x) { x |= x >> 16; x++; return x; +#endif } /* Compute the smallest power of 2 that is >= x. */ @@ -160,6 +213,27 @@ lg_floor(size_t x) { } #endif +BIT_UTIL_INLINE unsigned +lg_ceil(size_t x) { + return lg_floor(x) + ((x & (x - 1)) == 0 ? 0 : 1); +} + #undef BIT_UTIL_INLINE +/* A compile-time version of lg_floor and lg_ceil. */ +#define LG_FLOOR_1(x) 0 +#define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1)) +#define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2)) +#define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4)) +#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8)) +#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16)) +#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32)) +#if LG_SIZEOF_PTR == 2 +# define LG_FLOOR(x) LG_FLOOR_32((x)) +#else +# define LG_FLOOR(x) LG_FLOOR_64((x)) +#endif + +#define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1)) + #endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/bitmap.h b/deps/jemalloc/include/jemalloc/internal/bitmap.h index ac990290a..c3f9cb490 100644 --- a/deps/jemalloc/include/jemalloc/internal/bitmap.h +++ b/deps/jemalloc/include/jemalloc/internal/bitmap.h @@ -3,18 +3,18 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/bit_util.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef unsigned long bitmap_t; #define LG_SIZEOF_BITMAP LG_SIZEOF_LONG /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ -#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES +#if LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES) /* Maximum bitmap bit count is determined by maximum regions per slab. */ # define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS #else /* Maximum bitmap bit count is determined by number of extent size classes. */ -# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES +# define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES) #endif #define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) diff --git a/deps/jemalloc/include/jemalloc/internal/cache_bin.h b/deps/jemalloc/include/jemalloc/internal/cache_bin.h index 12f3ef2dd..d14556a3d 100644 --- a/deps/jemalloc/include/jemalloc/internal/cache_bin.h +++ b/deps/jemalloc/include/jemalloc/internal/cache_bin.h @@ -88,11 +88,21 @@ JEMALLOC_ALWAYS_INLINE void * cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { void *ret; - if (unlikely(bin->ncached == 0)) { - bin->low_water = -1; - *success = false; - return NULL; + bin->ncached--; + + /* + * Check for both bin->ncached == 0 and ncached < low_water + * in a single branch. + */ + if (unlikely(bin->ncached <= bin->low_water)) { + bin->low_water = bin->ncached; + if (bin->ncached == -1) { + bin->ncached = 0; + *success = false; + return NULL; + } } + /* * success (instead of ret) should be checked upon the return of this * function. We avoid checking (ret == NULL) because there is never a @@ -101,14 +111,21 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { * cacheline). */ *success = true; - ret = *(bin->avail - bin->ncached); - bin->ncached--; - - if (unlikely(bin->ncached < bin->low_water)) { - bin->low_water = bin->ncached; - } + ret = *(bin->avail - (bin->ncached + 1)); return ret; } +JEMALLOC_ALWAYS_INLINE bool +cache_bin_dalloc_easy(cache_bin_t *bin, cache_bin_info_t *bin_info, void *ptr) { + if (unlikely(bin->ncached == bin_info->ncached_max)) { + return false; + } + assert(bin->ncached < bin_info->ncached_max); + bin->ncached++; + *(bin->avail - bin->ncached) = ptr; + + return true; +} + #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/ctl.h b/deps/jemalloc/include/jemalloc/internal/ctl.h index d927d9480..1d1aacc6f 100644 --- a/deps/jemalloc/include/jemalloc/internal/ctl.h +++ b/deps/jemalloc/include/jemalloc/internal/ctl.h @@ -5,7 +5,7 @@ #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/stats.h" /* Maximum ctl tree depth. */ @@ -39,9 +39,12 @@ typedef struct ctl_arena_stats_s { uint64_t nmalloc_small; uint64_t ndalloc_small; uint64_t nrequests_small; + uint64_t nfills_small; + uint64_t nflushes_small; - bin_stats_t bstats[NBINS]; - arena_stats_large_t lstats[NSIZES - NBINS]; + bin_stats_t bstats[SC_NBINS]; + arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; + arena_stats_extents_t estats[SC_NPSIZES]; } ctl_arena_stats_t; typedef struct ctl_stats_s { diff --git a/deps/jemalloc/include/jemalloc/internal/emitter.h b/deps/jemalloc/include/jemalloc/internal/emitter.h index 3a2b2f7f2..542bc79c3 100644 --- a/deps/jemalloc/include/jemalloc/internal/emitter.h +++ b/deps/jemalloc/include/jemalloc/internal/emitter.h @@ -45,7 +45,9 @@ struct emitter_col_s { int int_val; unsigned unsigned_val; uint32_t uint32_val; + uint32_t uint32_t_val; uint64_t uint64_val; + uint64_t uint64_t_val; size_t size_val; ssize_t ssize_val; const char *str_val; @@ -60,17 +62,6 @@ struct emitter_row_s { ql_head(emitter_col_t) cols; }; -static inline void -emitter_row_init(emitter_row_t *row) { - ql_new(&row->cols); -} - -static inline void -emitter_col_init(emitter_col_t *col, emitter_row_t *row) { - ql_elm_new(col, link); - ql_tail_insert(&row->cols, col, link); -} - typedef struct emitter_s emitter_t; struct emitter_s { emitter_output_t output; @@ -80,18 +71,10 @@ struct emitter_s { int nesting_depth; /* True if we've already emitted a value at the given depth. */ bool item_at_depth; + /* True if we emitted a key and will emit corresponding value next. */ + bool emitted_key; }; -static inline void -emitter_init(emitter_t *emitter, emitter_output_t emitter_output, - void (*write_cb)(void *, const char *), void *cbopaque) { - emitter->output = emitter_output; - emitter->write_cb = write_cb; - emitter->cbopaque = cbopaque; - emitter->item_at_depth = false; - emitter->nesting_depth = 0; -} - /* Internal convenience function. Write to the emitter the given string. */ JEMALLOC_FORMAT_PRINTF(2, 3) static inline void @@ -103,22 +86,11 @@ emitter_printf(emitter_t *emitter, const char *format, ...) { va_end(ap); } -/* Write to the emitter the given string, but only in table mode. */ -JEMALLOC_FORMAT_PRINTF(2, 3) -static inline void -emitter_table_printf(emitter_t *emitter, const char *format, ...) { - if (emitter->output == emitter_output_table) { - va_list ap; - va_start(ap, format); - malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); - va_end(ap); - } -} - -static inline void +static inline const char * JEMALLOC_FORMAT_ARG(3) emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, emitter_justify_t justify, int width) { size_t written; + fmt_specifier++; if (justify == emitter_justify_none) { written = malloc_snprintf(out_fmt, out_size, "%%%s", fmt_specifier); @@ -131,6 +103,7 @@ emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, } /* Only happens in case of bad format string, which *we* choose. */ assert(written < out_size); + return out_fmt; } /* @@ -156,26 +129,27 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, char buf[BUF_SIZE]; #define EMIT_SIMPLE(type, format) \ - emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width); \ - emitter_printf(emitter, fmt, *(const type *)value); \ + emitter_printf(emitter, \ + emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width), \ + *(const type *)value); switch (value_type) { case emitter_type_bool: - emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); - emitter_printf(emitter, fmt, *(const bool *)value ? - "true" : "false"); + emitter_printf(emitter, + emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), + *(const bool *)value ? "true" : "false"); break; case emitter_type_int: - EMIT_SIMPLE(int, "d") + EMIT_SIMPLE(int, "%d") break; case emitter_type_unsigned: - EMIT_SIMPLE(unsigned, "u") + EMIT_SIMPLE(unsigned, "%u") break; case emitter_type_ssize: - EMIT_SIMPLE(ssize_t, "zd") + EMIT_SIMPLE(ssize_t, "%zd") break; case emitter_type_size: - EMIT_SIMPLE(size_t, "zu") + EMIT_SIMPLE(size_t, "%zu") break; case emitter_type_string: str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", @@ -185,17 +159,17 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, * anywhere near the fmt size. */ assert(str_written < BUF_SIZE); - emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); - emitter_printf(emitter, fmt, buf); + emitter_printf(emitter, + emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf); break; case emitter_type_uint32: - EMIT_SIMPLE(uint32_t, FMTu32) + EMIT_SIMPLE(uint32_t, "%" FMTu32) break; case emitter_type_uint64: - EMIT_SIMPLE(uint64_t, FMTu64) + EMIT_SIMPLE(uint64_t, "%" FMTu64) break; case emitter_type_title: - EMIT_SIMPLE(char *const, "s"); + EMIT_SIMPLE(char *const, "%s"); break; default: unreachable(); @@ -235,47 +209,143 @@ emitter_indent(emitter_t *emitter) { static inline void emitter_json_key_prefix(emitter_t *emitter) { + if (emitter->emitted_key) { + emitter->emitted_key = false; + return; + } emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : ""); emitter_indent(emitter); } -static inline void -emitter_begin(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth == 0); - emitter_printf(emitter, "{"); - emitter_nest_inc(emitter); - } else { - // tabular init - emitter_printf(emitter, "%s", ""); - } -} +/******************************************************************************/ +/* Public functions for emitter_t. */ static inline void -emitter_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth == 1); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n}\n"); - } +emitter_init(emitter_t *emitter, emitter_output_t emitter_output, + void (*write_cb)(void *, const char *), void *cbopaque) { + emitter->output = emitter_output; + emitter->write_cb = write_cb; + emitter->cbopaque = cbopaque; + emitter->item_at_depth = false; + emitter->emitted_key = false; + emitter->nesting_depth = 0; } -/* - * Note emits a different kv pair as well, but only in table mode. Omits the - * note if table_note_key is NULL. +/******************************************************************************/ +/* JSON public API. */ + +/* + * Emits a key (e.g. as appears in an object). The next json entity emitted will + * be the corresponding value. */ static inline void -emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, +emitter_json_key(emitter_t *emitter, const char *json_key) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "\"%s\": ", json_key); + emitter->emitted_key = true; + } +} + +static inline void +emitter_json_value(emitter_t *emitter, emitter_type_t value_type, + const void *value) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_print_value(emitter, emitter_justify_none, -1, + value_type, value); + emitter->item_at_depth = true; + } +} + +/* Shorthand for calling emitter_json_key and then emitter_json_value. */ +static inline void +emitter_json_kv(emitter_t *emitter, const char *json_key, + emitter_type_t value_type, const void *value) { + emitter_json_key(emitter, json_key); + emitter_json_value(emitter, value_type, value); +} + +static inline void +emitter_json_array_begin(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "["); + emitter_nest_inc(emitter); + } +} + +/* Shorthand for calling emitter_json_key and then emitter_json_array_begin. */ +static inline void +emitter_json_array_kv_begin(emitter_t *emitter, const char *json_key) { + emitter_json_key(emitter, json_key); + emitter_json_array_begin(emitter); +} + +static inline void +emitter_json_array_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "]"); + } +} + +static inline void +emitter_json_object_begin(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "{"); + emitter_nest_inc(emitter); + } +} + +/* Shorthand for calling emitter_json_key and then emitter_json_object_begin. */ +static inline void +emitter_json_object_kv_begin(emitter_t *emitter, const char *json_key) { + emitter_json_key(emitter, json_key); + emitter_json_object_begin(emitter); +} + +static inline void +emitter_json_object_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "}"); + } +} + + +/******************************************************************************/ +/* Table public API. */ + +static inline void +emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { + if (emitter->output == emitter_output_table) { + emitter_indent(emitter); + emitter_printf(emitter, "%s\n", table_key); + emitter_nest_inc(emitter); + } +} + +static inline void +emitter_table_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_table) { + emitter_nest_dec(emitter); + } +} + +static inline void +emitter_table_kv_note(emitter_t *emitter, const char *table_key, emitter_type_t value_type, const void *value, const char *table_note_key, emitter_type_t table_note_value_type, const void *table_note_value) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "\"%s\": ", json_key); - emitter_print_value(emitter, emitter_justify_none, -1, - value_type, value); - } else { + if (emitter->output == emitter_output_table) { emitter_indent(emitter); emitter_printf(emitter, "%s: ", table_key); emitter_print_value(emitter, emitter_justify_none, -1, @@ -292,130 +362,22 @@ emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, } static inline void -emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, +emitter_table_kv(emitter_t *emitter, const char *table_key, emitter_type_t value_type, const void *value) { - emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, + emitter_table_kv_note(emitter, table_key, value_type, value, NULL, emitter_type_bool, NULL); } -static inline void -emitter_json_kv(emitter_t *emitter, const char *json_key, - emitter_type_t value_type, const void *value) { - if (emitter->output == emitter_output_json) { - emitter_kv(emitter, json_key, NULL, value_type, value); - } -} +/* Write to the emitter the given string, but only in table mode. */ +JEMALLOC_FORMAT_PRINTF(2, 3) static inline void -emitter_table_kv(emitter_t *emitter, const char *table_key, - emitter_type_t value_type, const void *value) { +emitter_table_printf(emitter_t *emitter, const char *format, ...) { if (emitter->output == emitter_output_table) { - emitter_kv(emitter, NULL, table_key, value_type, value); - } -} - -static inline void -emitter_dict_begin(emitter_t *emitter, const char *json_key, - const char *table_header) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "\"%s\": {", json_key); - emitter_nest_inc(emitter); - } else { - emitter_indent(emitter); - emitter_printf(emitter, "%s\n", table_header); - emitter_nest_inc(emitter); - } -} - -static inline void -emitter_dict_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n"); - emitter_indent(emitter); - emitter_printf(emitter, "}"); - } else { - emitter_nest_dec(emitter); - } -} - -static inline void -emitter_json_dict_begin(emitter_t *emitter, const char *json_key) { - if (emitter->output == emitter_output_json) { - emitter_dict_begin(emitter, json_key, NULL); - } -} - -static inline void -emitter_json_dict_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - emitter_dict_end(emitter); - } -} - -static inline void -emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { - if (emitter->output == emitter_output_table) { - emitter_dict_begin(emitter, NULL, table_key); - } -} - -static inline void -emitter_table_dict_end(emitter_t *emitter) { - if (emitter->output == emitter_output_table) { - emitter_dict_end(emitter); - } -} - -static inline void -emitter_json_arr_begin(emitter_t *emitter, const char *json_key) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "\"%s\": [", json_key); - emitter_nest_inc(emitter); - } -} - -static inline void -emitter_json_arr_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n"); - emitter_indent(emitter); - emitter_printf(emitter, "]"); - } -} - -static inline void -emitter_json_arr_obj_begin(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "{"); - emitter_nest_inc(emitter); - } -} - -static inline void -emitter_json_arr_obj_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n"); - emitter_indent(emitter); - emitter_printf(emitter, "}"); - } -} - -static inline void -emitter_json_arr_value(emitter_t *emitter, emitter_type_t value_type, - const void *value) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_print_value(emitter, emitter_justify_none, -1, - value_type, value); + va_list ap; + va_start(ap, format); + malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); + va_end(ap); } } @@ -432,4 +394,93 @@ emitter_table_row(emitter_t *emitter, emitter_row_t *row) { emitter_table_printf(emitter, "\n"); } +static inline void +emitter_row_init(emitter_row_t *row) { + ql_new(&row->cols); +} + +static inline void +emitter_col_init(emitter_col_t *col, emitter_row_t *row) { + ql_elm_new(col, link); + ql_tail_insert(&row->cols, col, link); +} + + +/******************************************************************************/ +/* + * Generalized public API. Emits using either JSON or table, according to + * settings in the emitter_t. */ + +/* + * Note emits a different kv pair as well, but only in table mode. Omits the + * note if table_note_key is NULL. + */ +static inline void +emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, + emitter_type_t value_type, const void *value, + const char *table_note_key, emitter_type_t table_note_value_type, + const void *table_note_value) { + if (emitter->output == emitter_output_json) { + emitter_json_key(emitter, json_key); + emitter_json_value(emitter, value_type, value); + } else { + emitter_table_kv_note(emitter, table_key, value_type, value, + table_note_key, table_note_value_type, table_note_value); + } + emitter->item_at_depth = true; +} + +static inline void +emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, + emitter_type_t value_type, const void *value) { + emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, + emitter_type_bool, NULL); +} + +static inline void +emitter_dict_begin(emitter_t *emitter, const char *json_key, + const char *table_header) { + if (emitter->output == emitter_output_json) { + emitter_json_key(emitter, json_key); + emitter_json_object_begin(emitter); + } else { + emitter_table_dict_begin(emitter, table_header); + } +} + +static inline void +emitter_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_json_object_end(emitter); + } else { + emitter_table_dict_end(emitter); + } +} + +static inline void +emitter_begin(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth == 0); + emitter_printf(emitter, "{"); + emitter_nest_inc(emitter); + } else { + /* + * This guarantees that we always call write_cb at least once. + * This is useful if some invariant is established by each call + * to write_cb, but doesn't hold initially: e.g., some buffer + * holds a null-terminated string. + */ + emitter_printf(emitter, "%s", ""); + } +} + +static inline void +emitter_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth == 1); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n}\n"); + } +} + #endif /* JEMALLOC_INTERNAL_EMITTER_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/extent_externs.h b/deps/jemalloc/include/jemalloc/internal/extent_externs.h index b8a4d026c..8aba57633 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_externs.h @@ -24,13 +24,17 @@ size_t extent_size_quantize_floor(size_t size); size_t extent_size_quantize_ceil(size_t size); #endif -rb_proto(, extent_avail_, extent_tree_t, extent_t) +ph_proto(, extent_avail_, extent_tree_t, extent_t) ph_proto(, extent_heap_, extent_heap_t, extent_t) bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, bool delay_coalesce); extent_state_t extents_state_get(const extents_t *extents); size_t extents_npages_get(extents_t *extents); +/* Get the number of extents in the given page size index. */ +size_t extents_nextents_get(extents_t *extents, pszind_t ind); +/* Get the sum total bytes of the extents in the given page size index. */ +size_t extents_nbytes_get(extents_t *extents, pszind_t ind); extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, @@ -70,4 +74,10 @@ bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_boot(void); +void extent_util_stats_get(tsdn_t *tsdn, const void *ptr, + size_t *nfree, size_t *nregs, size_t *size); +void extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr, + size_t *nfree, size_t *nregs, size_t *size, + size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr); + #endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/extent_inlines.h b/deps/jemalloc/include/jemalloc/internal/extent_inlines.h index 77181df8d..77fa4c4a2 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_inlines.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_inlines.h @@ -6,6 +6,7 @@ #include "jemalloc/internal/pages.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ql.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" static inline void @@ -34,18 +35,19 @@ extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { (uintptr_t)extent2); } -static inline arena_t * -extent_arena_get(const extent_t *extent) { +static inline unsigned +extent_arena_ind_get(const extent_t *extent) { unsigned arena_ind = (unsigned)((extent->e_bits & EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); - /* - * The following check is omitted because we should never actually read - * a NULL arena pointer. - */ - if (false && arena_ind >= MALLOCX_ARENA_LIMIT) { - return NULL; - } assert(arena_ind < MALLOCX_ARENA_LIMIT); + + return arena_ind; +} + +static inline arena_t * +extent_arena_get(const extent_t *extent) { + unsigned arena_ind = extent_arena_ind_get(extent); + return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); } @@ -53,14 +55,14 @@ static inline szind_t extent_szind_get_maybe_invalid(const extent_t *extent) { szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> EXTENT_BITS_SZIND_SHIFT); - assert(szind <= NSIZES); + assert(szind <= SC_NSIZES); return szind; } static inline szind_t extent_szind_get(const extent_t *extent) { szind_t szind = extent_szind_get_maybe_invalid(extent); - assert(szind < NSIZES); /* Never call when "invalid". */ + assert(szind < SC_NSIZES); /* Never call when "invalid". */ return szind; } @@ -69,6 +71,14 @@ extent_usize_get(const extent_t *extent) { return sz_index2size(extent_szind_get(extent)); } +static inline unsigned +extent_binshard_get(const extent_t *extent) { + unsigned binshard = (unsigned)((extent->e_bits & + EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT); + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + return binshard; +} + static inline size_t extent_sn_get(const extent_t *extent) { return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> @@ -176,6 +186,11 @@ extent_prof_tctx_get(const extent_t *extent) { ATOMIC_ACQUIRE); } +static inline nstime_t +extent_prof_alloc_time_get(const extent_t *extent) { + return extent->e_alloc_time; +} + static inline void extent_arena_set(extent_t *extent, arena_t *arena) { unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << @@ -184,13 +199,21 @@ extent_arena_set(extent_t *extent, arena_t *arena) { ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); } +static inline void +extent_binshard_set(extent_t *extent, unsigned binshard) { + /* The assertion assumes szind is set already. */ + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) | + ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT); +} + static inline void extent_addr_set(extent_t *extent, void *addr) { extent->e_addr = addr; } static inline void -extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) { +extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { assert(extent_base_get(extent) == extent_addr_get(extent)); if (alignment < PAGE) { @@ -234,7 +257,7 @@ extent_bsize_set(extent_t *extent, size_t bsize) { static inline void extent_szind_set(extent_t *extent, szind_t szind) { - assert(szind <= NSIZES); /* NSIZES means "invalid". */ + assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); } @@ -246,6 +269,16 @@ extent_nfree_set(extent_t *extent, unsigned nfree) { ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); } +static inline void +extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) { + /* The assertion assumes szind is set already. */ + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + extent->e_bits = (extent->e_bits & + (~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) | + ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) | + ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); +} + static inline void extent_nfree_inc(extent_t *extent) { assert(extent_slab_get(extent)); @@ -258,6 +291,12 @@ extent_nfree_dec(extent_t *extent) { extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); } +static inline void +extent_nfree_sub(extent_t *extent, uint64_t n) { + assert(extent_slab_get(extent)); + extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT); +} + static inline void extent_sn_set(extent_t *extent, size_t sn) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | @@ -299,10 +338,35 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); } +static inline void +extent_prof_alloc_time_set(extent_t *extent, nstime_t t) { + nstime_copy(&extent->e_alloc_time, &t); +} + +static inline bool +extent_is_head_get(extent_t *extent) { + if (maps_coalesce) { + not_reached(); + } + + return (bool)((extent->e_bits & EXTENT_BITS_IS_HEAD_MASK) >> + EXTENT_BITS_IS_HEAD_SHIFT); +} + +static inline void +extent_is_head_set(extent_t *extent, bool is_head) { + if (maps_coalesce) { + not_reached(); + } + + extent->e_bits = (extent->e_bits & ~EXTENT_BITS_IS_HEAD_MASK) | + ((uint64_t)is_head << EXTENT_BITS_IS_HEAD_SHIFT); +} + static inline void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, - bool committed, bool dumpable) { + bool committed, bool dumpable, extent_head_state_t is_head) { assert(addr == PAGE_ADDR2BASE(addr) || !slab); extent_arena_set(extent, arena); @@ -316,6 +380,10 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, extent_committed_set(extent, committed); extent_dumpable_set(extent, dumpable); ql_elm_new(extent, ql_link); + if (!maps_coalesce) { + extent_is_head_set(extent, (is_head == EXTENT_IS_HEAD) ? true : + false); + } if (config_prof) { extent_prof_tctx_set(extent, NULL); } @@ -327,7 +395,7 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { extent_addr_set(extent, addr); extent_bsize_set(extent, bsize); extent_slab_set(extent, false); - extent_szind_set(extent, NSIZES); + extent_szind_set(extent, SC_NSIZES); extent_sn_set(extent, sn); extent_state_set(extent, extent_state_active); extent_zeroed_set(extent, true); diff --git a/deps/jemalloc/include/jemalloc/internal/extent_structs.h b/deps/jemalloc/include/jemalloc/internal/extent_structs.h index 4873b9e9e..767cd8930 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_structs.h @@ -2,11 +2,12 @@ #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/ql.h" #include "jemalloc/internal/ph.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef enum { extent_state_active = 0, @@ -28,9 +29,10 @@ struct extent_s { * t: state * i: szind * f: nfree + * s: bin_shard * n: sn * - * nnnnnnnn ... nnnnffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa + * nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa * * arena_ind: Arena from which this extent came, or all 1 bits if * unassociated. @@ -75,6 +77,8 @@ struct extent_s { * * nfree: Number of free regions in slab. * + * bin_shard: the shard of the bin from which this extent came. + * * sn: Serial number (potentially non-unique). * * Serial numbers may wrap around if !opt_retain, but as long as @@ -112,7 +116,7 @@ struct extent_s { #define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT) #define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT) -#define EXTENT_BITS_SZIND_WIDTH LG_CEIL_NSIZES +#define EXTENT_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) #define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT) #define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT) @@ -120,7 +124,15 @@ struct extent_s { #define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT) #define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT) -#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) +#define EXTENT_BITS_BINSHARD_WIDTH 6 +#define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) +#define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT) + +#define EXTENT_BITS_IS_HEAD_WIDTH 1 +#define EXTENT_BITS_IS_HEAD_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT) +#define EXTENT_BITS_IS_HEAD_MASK MASK(EXTENT_BITS_IS_HEAD_WIDTH, EXTENT_BITS_IS_HEAD_SHIFT) + +#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_IS_HEAD_WIDTH + EXTENT_BITS_IS_HEAD_SHIFT) #define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) /* Pointer to the extent that this structure is responsible for. */ @@ -160,11 +172,13 @@ struct extent_s { /* Small region slab metadata. */ arena_slab_data_t e_slab_data; - /* - * Profile counters, used for large objects. Points to a - * prof_tctx_t. - */ - atomic_p_t e_prof_tctx; + /* Profiling data, used for large objects. */ + struct { + /* Time when this was allocated. */ + nstime_t e_alloc_time; + /* Points to a prof_tctx_t. */ + atomic_p_t e_prof_tctx; + }; }; }; typedef ql_head(extent_t) extent_list_t; @@ -180,14 +194,16 @@ struct extents_s { * * Synchronization: mtx. */ - extent_heap_t heaps[NPSIZES+1]; + extent_heap_t heaps[SC_NPSIZES + 1]; + atomic_zu_t nextents[SC_NPSIZES + 1]; + atomic_zu_t nbytes[SC_NPSIZES + 1]; /* * Bitmap for which set bits correspond to non-empty heaps. * * Synchronization: mtx. */ - bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)]; + bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)]; /* * LRU of all extents in heaps. @@ -216,4 +232,25 @@ struct extents_s { bool delay_coalesce; }; +/* + * The following two structs are for experimental purposes. See + * experimental_utilization_query_ctl and + * experimental_utilization_batch_query_ctl in src/ctl.c. + */ + +struct extent_util_stats_s { + size_t nfree; + size_t nregs; + size_t size; +}; + +struct extent_util_stats_verbose_s { + void *slabcur_addr; + size_t nfree; + size_t nregs; + size_t size; + size_t bin_nfree; + size_t bin_nregs; +}; + #endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/extent_types.h b/deps/jemalloc/include/jemalloc/internal/extent_types.h index c0561d99f..96925cf95 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_types.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_types.h @@ -4,9 +4,10 @@ typedef struct extent_s extent_t; typedef struct extents_s extents_t; -#define EXTENT_HOOKS_INITIALIZER NULL +typedef struct extent_util_stats_s extent_util_stats_t; +typedef struct extent_util_stats_verbose_s extent_util_stats_verbose_t; -#define EXTENT_GROW_MAX_PIND (NPSIZES - 1) +#define EXTENT_HOOKS_INITIALIZER NULL /* * When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit) @@ -14,4 +15,9 @@ typedef struct extents_s extents_t; */ #define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6 +typedef enum { + EXTENT_NOT_HEAD, + EXTENT_IS_HEAD /* Only relevant for Windows && opt.retain. */ +} extent_head_state_t; + #endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/hash.h b/deps/jemalloc/include/jemalloc/internal/hash.h index dcfc992df..0270034e8 100644 --- a/deps/jemalloc/include/jemalloc/internal/hash.h +++ b/deps/jemalloc/include/jemalloc/internal/hash.h @@ -104,8 +104,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) { uint32_t k1 = 0; switch (len & 3) { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; + case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; } @@ -119,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) { return h1; } -UNUSED static inline void +static inline void hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) { const uint8_t * data = (const uint8_t *) key; @@ -177,28 +177,29 @@ hash_x86_128(const void *key, const int len, uint32_t seed, uint32_t k4 = 0; switch (len & 15) { - case 15: k4 ^= tail[14] << 16; - case 14: k4 ^= tail[13] << 8; + case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH + case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH case 13: k4 ^= tail[12] << 0; k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; - - case 12: k3 ^= tail[11] << 24; - case 11: k3 ^= tail[10] << 16; - case 10: k3 ^= tail[ 9] << 8; + JEMALLOC_FALLTHROUGH + case 12: k3 ^= tail[11] << 24; JEMALLOC_FALLTHROUGH + case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH + case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH case 9: k3 ^= tail[ 8] << 0; k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; - - case 8: k2 ^= tail[ 7] << 24; - case 7: k2 ^= tail[ 6] << 16; - case 6: k2 ^= tail[ 5] << 8; + JEMALLOC_FALLTHROUGH + case 8: k2 ^= tail[ 7] << 24; JEMALLOC_FALLTHROUGH + case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH + case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH case 5: k2 ^= tail[ 4] << 0; k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; - - case 4: k1 ^= tail[ 3] << 24; - case 3: k1 ^= tail[ 2] << 16; - case 2: k1 ^= tail[ 1] << 8; + JEMALLOC_FALLTHROUGH + case 4: k1 ^= tail[ 3] << 24; JEMALLOC_FALLTHROUGH + case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= tail[ 0] << 0; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + JEMALLOC_FALLTHROUGH } } @@ -220,7 +221,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed, r_out[1] = (((uint64_t) h4) << 32) | h3; } -UNUSED static inline void +static inline void hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) { const uint8_t *data = (const uint8_t *) key; @@ -260,22 +261,22 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t k2 = 0; switch (len & 15) { - case 15: k2 ^= ((uint64_t)(tail[14])) << 48; /* falls through */ - case 14: k2 ^= ((uint64_t)(tail[13])) << 40; /* falls through */ - case 13: k2 ^= ((uint64_t)(tail[12])) << 32; /* falls through */ - case 12: k2 ^= ((uint64_t)(tail[11])) << 24; /* falls through */ - case 11: k2 ^= ((uint64_t)(tail[10])) << 16; /* falls through */ - case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; /* falls through */ + case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH + case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH + case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH + case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH + case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH + case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; - /* falls through */ - case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; /* falls through */ - case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; /* falls through */ - case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; /* falls through */ - case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; /* falls through */ - case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; /* falls through */ - case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; /* falls through */ - case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; /* falls through */ + JEMALLOC_FALLTHROUGH + case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH + case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH + case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH + case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH + case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH + case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; } diff --git a/deps/jemalloc/include/jemalloc/internal/hook.h b/deps/jemalloc/include/jemalloc/internal/hook.h new file mode 100644 index 000000000..ee246b1e0 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/hook.h @@ -0,0 +1,163 @@ +#ifndef JEMALLOC_INTERNAL_HOOK_H +#define JEMALLOC_INTERNAL_HOOK_H + +#include "jemalloc/internal/tsd.h" + +/* + * This API is *extremely* experimental, and may get ripped out, changed in API- + * and ABI-incompatible ways, be insufficiently or incorrectly documented, etc. + * + * It allows hooking the stateful parts of the API to see changes as they + * happen. + * + * Allocation hooks are called after the allocation is done, free hooks are + * called before the free is done, and expand hooks are called after the + * allocation is expanded. + * + * For realloc and rallocx, if the expansion happens in place, the expansion + * hook is called. If it is moved, then the alloc hook is called on the new + * location, and then the free hook is called on the old location (i.e. both + * hooks are invoked in between the alloc and the dalloc). + * + * If we return NULL from OOM, then usize might not be trustworthy. Calling + * realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0) + * only calls the free hook. (Calling realloc(NULL, 0) is treated as malloc(0), + * and only calls the alloc hook). + * + * Reentrancy: + * Reentrancy is guarded against from within the hook implementation. If you + * call allocator functions from within a hook, the hooks will not be invoked + * again. + * Threading: + * The installation of a hook synchronizes with all its uses. If you can + * prove the installation of a hook happens-before a jemalloc entry point, + * then the hook will get invoked (unless there's a racing removal). + * + * Hook insertion appears to be atomic at a per-thread level (i.e. if a thread + * allocates and has the alloc hook invoked, then a subsequent free on the + * same thread will also have the free hook invoked). + * + * The *removal* of a hook does *not* block until all threads are done with + * the hook. Hook authors have to be resilient to this, and need some + * out-of-band mechanism for cleaning up any dynamically allocated memory + * associated with their hook. + * Ordering: + * Order of hook execution is unspecified, and may be different than insertion + * order. + */ + +#define HOOK_MAX 4 + +enum hook_alloc_e { + hook_alloc_malloc, + hook_alloc_posix_memalign, + hook_alloc_aligned_alloc, + hook_alloc_calloc, + hook_alloc_memalign, + hook_alloc_valloc, + hook_alloc_mallocx, + + /* The reallocating functions have both alloc and dalloc variants */ + hook_alloc_realloc, + hook_alloc_rallocx, +}; +/* + * We put the enum typedef after the enum, since this file may get included by + * jemalloc_cpp.cpp, and C++ disallows enum forward declarations. + */ +typedef enum hook_alloc_e hook_alloc_t; + +enum hook_dalloc_e { + hook_dalloc_free, + hook_dalloc_dallocx, + hook_dalloc_sdallocx, + + /* + * The dalloc halves of reallocation (not called if in-place expansion + * happens). + */ + hook_dalloc_realloc, + hook_dalloc_rallocx, +}; +typedef enum hook_dalloc_e hook_dalloc_t; + + +enum hook_expand_e { + hook_expand_realloc, + hook_expand_rallocx, + hook_expand_xallocx, +}; +typedef enum hook_expand_e hook_expand_t; + +typedef void (*hook_alloc)( + void *extra, hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]); + +typedef void (*hook_dalloc)( + void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]); + +typedef void (*hook_expand)( + void *extra, hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); + +typedef struct hooks_s hooks_t; +struct hooks_s { + hook_alloc alloc_hook; + hook_dalloc dalloc_hook; + hook_expand expand_hook; + void *extra; +}; + +/* + * Begin implementation details; everything above this point might one day live + * in a public API. Everything below this point never will. + */ + +/* + * The realloc pathways haven't gotten any refactoring love in a while, and it's + * fairly difficult to pass information from the entry point to the hooks. We + * put the informaiton the hooks will need into a struct to encapsulate + * everything. + * + * Much of these pathways are force-inlined, so that the compiler can avoid + * materializing this struct until we hit an extern arena function. For fairly + * goofy reasons, *many* of the realloc paths hit an extern arena function. + * These paths are cold enough that it doesn't matter; eventually, we should + * rewrite the realloc code to make the expand-in-place and the + * free-then-realloc paths more orthogonal, at which point we don't need to + * spread the hook logic all over the place. + */ +typedef struct hook_ralloc_args_s hook_ralloc_args_t; +struct hook_ralloc_args_s { + /* I.e. as opposed to rallocx. */ + bool is_realloc; + /* + * The expand hook takes 4 arguments, even if only 3 are actually used; + * we add an extra one in case the user decides to memcpy without + * looking too closely at the hooked function. + */ + uintptr_t args[4]; +}; + +/* + * Returns an opaque handle to be used when removing the hook. NULL means that + * we couldn't install the hook. + */ +bool hook_boot(); + +void *hook_install(tsdn_t *tsdn, hooks_t *hooks); +/* Uninstalls the hook with the handle previously returned from hook_install. */ +void hook_remove(tsdn_t *tsdn, void *opaque); + +/* Hooks */ + +void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]); + +void hook_invoke_dalloc(hook_dalloc_t type, void *address, + uintptr_t args_raw[3]); + +void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); + +#endif /* JEMALLOC_INTERNAL_HOOK_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/hooks.h b/deps/jemalloc/include/jemalloc/internal/hooks.h deleted file mode 100644 index cd49afcb0..000000000 --- a/deps/jemalloc/include/jemalloc/internal/hooks.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_HOOKS_H -#define JEMALLOC_INTERNAL_HOOKS_H - -extern JEMALLOC_EXPORT void (*hooks_arena_new_hook)(); -extern JEMALLOC_EXPORT void (*hooks_libc_hook)(); - -#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) - -#define open JEMALLOC_HOOK(open, hooks_libc_hook) -#define read JEMALLOC_HOOK(read, hooks_libc_hook) -#define write JEMALLOC_HOOK(write, hooks_libc_hook) -#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook) -#define close JEMALLOC_HOOK(close, hooks_libc_hook) -#define creat JEMALLOC_HOOK(creat, hooks_libc_hook) -#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook) -/* Note that this is undef'd and re-define'd in src/prof.c. */ -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) - -#endif /* JEMALLOC_INTERNAL_HOOKS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h index be70df510..7d6053e21 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h @@ -31,6 +31,9 @@ # include # endif # include +# ifdef __FreeBSD__ +# include +# endif # include # ifdef JEMALLOC_OS_UNFAIR_LOCK # include diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in index 8dad9a1db..c442a2191 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -48,25 +48,13 @@ /* Defined if GCC __atomic atomics are available. */ #undef JEMALLOC_GCC_ATOMIC_ATOMICS +/* and the 8-bit variant support. */ +#undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ #undef JEMALLOC_GCC_SYNC_ATOMICS - -/* - * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and - * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 - -/* - * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and - * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 +/* and the 8-bit variant support. */ +#undef JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -78,12 +66,6 @@ */ #undef JEMALLOC_OS_UNFAIR_LOCK -/* - * Defined if OSSpin*() functions are available, as provided by Darwin, and - * documented in the spinlock(3) manual page. - */ -#undef JEMALLOC_OSSPIN - /* Defined if syscall(2) is usable. */ #undef JEMALLOC_USE_SYSCALL @@ -153,6 +135,9 @@ /* JEMALLOC_STATS enables statistics calculation. */ #undef JEMALLOC_STATS +/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */ +#undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API + /* JEMALLOC_PROF enables allocation profiling. */ #undef JEMALLOC_PROF @@ -233,6 +218,12 @@ #undef JEMALLOC_INTERNAL_FFSL #undef JEMALLOC_INTERNAL_FFS +/* + * popcount*() functions to use for bitmapping. + */ +#undef JEMALLOC_INTERNAL_POPCOUNTL +#undef JEMALLOC_INTERNAL_POPCOUNT + /* * If defined, explicitly attempt to more uniformly distribute large allocation * pointer alignments across all cache indices. @@ -245,6 +236,12 @@ */ #undef JEMALLOC_LOG +/* + * If defined, use readlinkat() (instead of readlink()) to follow + * /etc/malloc_conf. + */ +#undef JEMALLOC_READLINKAT + /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ @@ -363,4 +360,7 @@ */ #undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* Performs additional safety checks when defined. */ +#undef JEMALLOC_OPT_SAFETY_CHECKS + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h index e10fb275d..d291170be 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h @@ -2,7 +2,6 @@ #define JEMALLOC_INTERNAL_EXTERNS_H #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/tsd_types.h" /* TSD checks this to set thread local slow state accordingly. */ @@ -11,6 +10,7 @@ extern bool malloc_slow; /* Run-time options. */ extern bool opt_abort; extern bool opt_abort_conf; +extern bool opt_confirm_conf; extern const char *opt_junk; extern bool opt_junk_alloc; extern bool opt_junk_free; @@ -25,6 +25,9 @@ extern unsigned ncpus; /* Number of arenas used for automatic multiplexing of threads and arenas. */ extern unsigned narenas_auto; +/* Base index for manual arenas. */ +extern unsigned manual_arena_base; + /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. @@ -49,5 +52,6 @@ void jemalloc_prefork(void); void jemalloc_postfork_parent(void); void jemalloc_postfork_child(void); bool malloc_initialized(void); +void je_sdallocx_noflags(void *ptr, size_t size); #endif /* JEMALLOC_INTERNAL_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h index c6a1f7eb2..ddde9b4e6 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -4,13 +4,15 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/ticker.h" JEMALLOC_ALWAYS_INLINE malloc_cpuid_t malloc_getcpu(void) { assert(have_percpu_arena); -#if defined(JEMALLOC_HAVE_SCHED_GETCPU) +#if defined(_WIN32) + return GetCurrentProcessorNumber(); +#elif defined(JEMALLOC_HAVE_SCHED_GETCPU) return (malloc_cpuid_t)sched_getcpu(); #else not_reached(); @@ -108,14 +110,14 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) { JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_small_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind < NBINS); + assert(binind < SC_NBINS); return &tcache->bins_small[binind]; } JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_large_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind >= NBINS &&binind < nhbins); - return &tcache->bins_large[binind - NBINS]; + assert(binind >= SC_NBINS &&binind < nhbins); + return &tcache->bins_large[binind - SC_NBINS]; } JEMALLOC_ALWAYS_INLINE bool @@ -156,7 +158,7 @@ pre_reentrancy(tsd_t *tsd, arena_t *arena) { if (fast) { /* Prepare slow path for reentrancy. */ tsd_slow_update(tsd); - assert(tsd->state == tsd_state_nominal_slow); + assert(tsd_state_get(tsd) == tsd_state_nominal_slow); } } diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h index 2e76e5d8f..70d6e5788 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -71,7 +71,8 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) { static inline bool arena_is_auto(arena_t *arena) { assert(narenas_auto > 0); - return (arena_ind_get(arena) < narenas_auto); + + return (arena_ind_get(arena) < manual_arena_base); } JEMALLOC_ALWAYS_INLINE extent_t * diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 2685802b8..b6e2f8c6d 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_INLINES_C_H #define JEMALLOC_INTERNAL_INLINES_C_H +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/witness.h" @@ -42,7 +43,6 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path) { void *ret; - assert(size != 0); assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { @@ -133,31 +133,20 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache, - arena_t *arena) { + size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, + hook_ralloc_args_t *hook_args) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); void *p; size_t usize, copysize; - usize = sz_sa2u(size + extra, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + usize = sz_sa2u(size, alignment); + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); if (p == NULL) { - if (extra == 0) { - return NULL; - } - /* Try again, without extra this time. */ - usize = sz_sa2u(size, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { - return NULL; - } - p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); - if (p == NULL) { - return NULL; - } + return NULL; } /* * Copy at most size bytes (not size+extra), since the caller has no @@ -165,13 +154,26 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, p, (uintptr_t)p, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); return p; } +/* + * is_realloc threads through the knowledge of whether or not this call comes + * from je_realloc (as opposed to je_rallocx); this ensures that we pass the + * correct entry point into any hooks. + * Note that these functions are all force-inlined, so no actual bool gets + * passed-around anywhere. + */ JEMALLOC_ALWAYS_INLINE void * iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, tcache_t *tcache, arena_t *arena) { + bool zero, tcache_t *tcache, arena_t *arena, hook_ralloc_args_t *hook_args) +{ assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -183,24 +185,24 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, * Existing object alignment is inadequate; allocate new space * and copy. */ - return iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment, - zero, tcache, arena); + return iralloct_realign(tsdn, ptr, oldsize, size, alignment, + zero, tcache, arena, hook_args); } return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, - tcache); + tcache, hook_args); } JEMALLOC_ALWAYS_INLINE void * iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero) { + bool zero, hook_ralloc_args_t *hook_args) { return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, - tcache_get(tsd), NULL); + tcache_get(tsd), NULL, hook_args); } JEMALLOC_ALWAYS_INLINE bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) { + size_t alignment, bool zero, size_t *newsize) { assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -209,10 +211,12 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { /* Existing object alignment is inadequate. */ + *newsize = oldsize; return true; } - return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero); + return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero, + newsize); } JEMALLOC_ALWAYS_INLINE int @@ -228,18 +232,20 @@ iget_defrag_hint(tsdn_t *tsdn, void* ptr) { extent_t *slab = iealloc(tsdn, ptr); arena_t *arena = extent_arena_get(slab); szind_t binind = extent_szind_get(slab); - bin_t *bin = &arena->bins[binind]; - malloc_mutex_lock(tsdn, &bin->lock); + unsigned shardind = extent_binshard_get(slab); + bins_t *bin = &arena->bins[binind]; + bin_t* binshard = &bin->bin_shards[shardind]; + malloc_mutex_lock(tsdn, &binshard->lock); /* don't bother moving allocations from the slab currently used for new allocations */ - if (slab != bin->slabcur) { + if (slab != binshard->slabcur) { int free_in_slab = extent_nfree_get(slab); if (free_in_slab) { const bin_info_t *bin_info = &bin_infos[binind]; - int curslabs = bin->stats.curslabs; - size_t curregs = bin->stats.curregs; - if (bin->slabcur) { + int curslabs = binshard->stats.curslabs; + size_t curregs = binshard->stats.curregs; + if (binshard->slabcur) { /* remove slabcur from the overall utilization */ - curregs -= bin_info->nregs - extent_nfree_get(bin->slabcur); + curregs -= bin_info->nregs - extent_nfree_get(binshard->slabcur); curslabs -= 1; } /* Compare the utilization ratio of the slab in question to the total average, @@ -249,7 +255,7 @@ iget_defrag_hint(tsdn_t *tsdn, void* ptr) { defrag = (bin_info->nregs - free_in_slab) * curslabs <= curregs; } } - malloc_mutex_unlock(tsdn, &bin->lock); + malloc_mutex_unlock(tsdn, &binshard->lock); } return defrag; } diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h index ed75d3768..d8ea06f6d 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -30,7 +30,7 @@ # define restrict #endif -/* Various function pointers are statick and immutable except during testing. */ +/* Various function pointers are static and immutable except during testing. */ #ifdef JEMALLOC_JET # define JET_MUTABLE #else @@ -40,4 +40,75 @@ #define JEMALLOC_VA_ARGS_HEAD(head, ...) head #define JEMALLOC_VA_ARGS_TAIL(head, ...) __VA_ARGS__ +#if (defined(__GNUC__) || defined(__GNUG__)) && !defined(__clang__) \ + && defined(JEMALLOC_HAVE_ATTR) && (__GNUC__ >= 7) +#define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough); +#else +#define JEMALLOC_FALLTHROUGH /* falls through */ +#endif + +/* Diagnostic suppression macros */ +#if defined(_MSC_VER) && !defined(__clang__) +# define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push)) +# define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop)) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable:W)) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +/* #pragma GCC diagnostic first appeared in gcc 4.6. */ +#elif (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && \ + (__GNUC_MINOR__ > 5)))) || defined(__clang__) +/* + * The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang + * diagnostic suppression macros and should not be used anywhere else. + */ +# define JEMALLOC_PRAGMA__(X) _Pragma(#X) +# define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push) +# define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) \ + JEMALLOC_PRAGMA__(GCC diagnostic ignored W) + +/* + * The -Wmissing-field-initializers warning is buggy in GCC versions < 5.1 and + * all clang versions up to version 7 (currently trunk, unreleased). This macro + * suppresses the warning for the affected compiler versions only. + */ +# if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) || \ + defined(__clang__) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wmissing-field-initializers") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# endif + +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits") +# define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter") +# if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# endif +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS \ + JEMALLOC_DIAGNOSTIC_PUSH \ + JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER +#else +# define JEMALLOC_DIAGNOSTIC_PUSH +# define JEMALLOC_DIAGNOSTIC_POP +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +#endif + +/* + * Disables spurious diagnostics for all headers. Since these headers are not + * included by users directly, it does not affect their diagnostic settings. + */ +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + #endif /* JEMALLOC_INTERNAL_MACROS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h index 1b750b122..e296c5a7e 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_TYPES_H #define JEMALLOC_INTERNAL_TYPES_H +#include "jemalloc/internal/quantum.h" + /* Page size index type. */ typedef unsigned pszind_t; @@ -50,79 +52,6 @@ typedef int malloc_cpuid_t; /* Smallest size class to support. */ #define TINY_MIN (1U << LG_TINY_MIN) -/* - * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size - * classes). - */ -#ifndef LG_QUANTUM -# if (defined(__i386__) || defined(_M_IX86)) -# define LG_QUANTUM 4 -# endif -# ifdef __ia64__ -# define LG_QUANTUM 4 -# endif -# ifdef __alpha__ -# define LG_QUANTUM 4 -# endif -# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) -# define LG_QUANTUM 4 -# endif -# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) -# define LG_QUANTUM 4 -# endif -# ifdef __arm__ -# define LG_QUANTUM 3 -# endif -# ifdef __aarch64__ -# define LG_QUANTUM 4 -# endif -# ifdef __hppa__ -# define LG_QUANTUM 4 -# endif -# ifdef __m68k__ -# define LG_QUANTUM 3 -# endif -# ifdef __mips__ -# define LG_QUANTUM 3 -# endif -# ifdef __nios2__ -# define LG_QUANTUM 3 -# endif -# ifdef __or1k__ -# define LG_QUANTUM 3 -# endif -# ifdef __powerpc__ -# define LG_QUANTUM 4 -# endif -# if defined(__riscv) || defined(__riscv__) -# define LG_QUANTUM 4 -# endif -# ifdef __s390__ -# define LG_QUANTUM 4 -# endif -# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ - defined(__SH4_SINGLE_ONLY__)) -# define LG_QUANTUM 4 -# endif -# ifdef __tile__ -# define LG_QUANTUM 4 -# endif -# ifdef __le32__ -# define LG_QUANTUM 4 -# endif -# ifndef LG_QUANTUM -# error "Unknown minimum alignment for architecture; specify via " - "--with-lg-quantum" -# endif -#endif - -#define QUANTUM ((size_t)(1U << LG_QUANTUM)) -#define QUANTUM_MASK (QUANTUM - 1) - -/* Return the smallest quantum multiple that is >= a. */ -#define QUANTUM_CEILING(a) \ - (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) - #define LONG ((size_t)(1U << LG_SIZEOF_LONG)) #define LONG_MASK (LONG - 1) diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in index e621fbc85..3418cbfa2 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in @@ -21,7 +21,7 @@ # include "../jemalloc@install_suffix@.h" #endif -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#if defined(JEMALLOC_OSATOMIC) #include #endif @@ -45,7 +45,7 @@ # include "jemalloc/internal/private_namespace_jet.h" # endif #endif -#include "jemalloc/internal/hooks.h" +#include "jemalloc/internal/test_hooks.h" #ifdef JEMALLOC_DEFINE_MADVISE_FREE # define JEMALLOC_MADV_FREE 8 @@ -161,7 +161,26 @@ static const bool config_log = false #endif ; -#ifdef JEMALLOC_HAVE_SCHED_GETCPU +/* + * Are extra safety checks enabled; things like checking the size of sized + * deallocations, double-frees, etc. + */ +static const bool config_opt_safety_checks = +#ifdef JEMALLOC_OPT_SAFETY_CHECKS + true +#elif defined(JEMALLOC_DEBUG) + /* + * This lets us only guard safety checks by one flag instead of two; fast + * checks can guard solely by config_opt_safety_checks and run in debug mode + * too. + */ + true +#else + false +#endif + ; + +#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA #endif diff --git a/deps/jemalloc/include/jemalloc/internal/large_externs.h b/deps/jemalloc/include/jemalloc/internal/large_externs.h index 3f36282cd..a05019e8a 100644 --- a/deps/jemalloc/include/jemalloc/internal/large_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/large_externs.h @@ -1,13 +1,16 @@ #ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H #define JEMALLOC_INTERNAL_LARGE_EXTERNS_H +#include "jemalloc/internal/hook.h" + void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero); void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero); bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t usize_max, bool zero); -void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, - size_t alignment, bool zero, tcache_t *tcache); +void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, + size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args); typedef void (large_dalloc_junk_t)(void *, size_t); extern large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk; @@ -23,4 +26,7 @@ prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent); void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx); void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent); +nstime_t large_prof_alloc_time_get(const extent_t *extent); +void large_prof_alloc_time_set(extent_t *extent, nstime_t time); + #endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/malloc_io.h b/deps/jemalloc/include/jemalloc/internal/malloc_io.h index bfe556b52..1d1a414e0 100644 --- a/deps/jemalloc/include/jemalloc/internal/malloc_io.h +++ b/deps/jemalloc/include/jemalloc/internal/malloc_io.h @@ -54,7 +54,7 @@ size_t malloc_vsnprintf(char *str, size_t size, const char *format, size_t malloc_snprintf(char *str, size_t size, const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); /* - * The caller can set write_cb and cbopaque to null to choose to print with the + * The caller can set write_cb to null to choose to print with the * je_malloc_message hook. */ void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, diff --git a/deps/jemalloc/include/jemalloc/internal/mutex.h b/deps/jemalloc/include/jemalloc/internal/mutex.h index 6520c2512..7c24f0725 100644 --- a/deps/jemalloc/include/jemalloc/internal/mutex.h +++ b/deps/jemalloc/include/jemalloc/internal/mutex.h @@ -37,14 +37,17 @@ struct malloc_mutex_s { # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock lock; -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLock lock; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) pthread_mutex_t lock; malloc_mutex_t *postponed_next; #else pthread_mutex_t lock; #endif + /* + * Hint flag to avoid exclusive cache line contention + * during spin waiting + */ + atomic_b_t locked; }; /* * We only touch witness when configured w/ debug. However we @@ -84,10 +87,6 @@ struct malloc_mutex_s { # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) -#elif (defined(JEMALLOC_OSSPIN)) -# define MALLOC_MUTEX_LOCK(m) OSSpinLockLock(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) OSSpinLockUnlock(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (!OSSpinLockTry(&(m)->lock)) #else # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) @@ -101,22 +100,37 @@ struct malloc_mutex_s { #ifdef _WIN32 # define MALLOC_MUTEX_INITIALIZER #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -#elif (defined(JEMALLOC_OSSPIN)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, 0}}, \ +# if defined(JEMALLOC_DEBUG) +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif #elif (defined(JEMALLOC_MUTEX_INIT_CB)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# if (defined(JEMALLOC_DEBUG)) +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif + #else # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT +# if defined(JEMALLOC_DEBUG) # define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif #endif #ifdef JEMALLOC_LAZY_LOCK @@ -139,6 +153,7 @@ void malloc_mutex_lock_slow(malloc_mutex_t *mutex); static inline void malloc_mutex_lock_final(malloc_mutex_t *mutex) { MALLOC_MUTEX_LOCK(mutex); + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } static inline bool @@ -164,6 +179,7 @@ malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); return true; } mutex_owner_stats_update(tsdn, mutex); @@ -203,6 +219,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { malloc_mutex_lock_slow(mutex); + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } mutex_owner_stats_update(tsdn, mutex); } @@ -211,6 +228,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { static inline void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { + atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { MALLOC_MUTEX_UNLOCK(mutex); @@ -245,4 +263,26 @@ malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); } +static inline void +malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, + malloc_mutex_t *mutex) { + mutex_prof_data_t *source = &mutex->prof_data; + /* Can only read holding the mutex. */ + malloc_mutex_assert_owner(tsdn, mutex); + + nstime_add(&data->tot_wait_time, &source->tot_wait_time); + if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { + nstime_copy(&data->max_wait_time, &source->max_wait_time); + } + data->n_wait_times += source->n_wait_times; + data->n_spin_acquired += source->n_spin_acquired; + if (data->max_n_thds < source->max_n_thds) { + data->max_n_thds = source->max_n_thds; + } + /* n_wait_thds is not reported. */ + atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); + data->n_owner_switches += source->n_owner_switches; + data->n_lock_ops += source->n_lock_ops; +} + #endif /* JEMALLOC_INTERNAL_MUTEX_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/mutex_prof.h b/deps/jemalloc/include/jemalloc/internal/mutex_prof.h index ce183d335..2cb8fb0cb 100644 --- a/deps/jemalloc/include/jemalloc/internal/mutex_prof.h +++ b/deps/jemalloc/include/jemalloc/internal/mutex_prof.h @@ -35,22 +35,31 @@ typedef enum { mutex_prof_num_arena_mutexes } mutex_prof_arena_ind_t; +/* + * The forth parameter is a boolean value that is true for derived rate counters + * and false for real ones. + */ #define MUTEX_PROF_UINT64_COUNTERS \ - OP(num_ops, uint64_t, "n_lock_ops") \ - OP(num_wait, uint64_t, "n_waiting") \ - OP(num_spin_acq, uint64_t, "n_spin_acq") \ - OP(num_owner_switch, uint64_t, "n_owner_switch") \ - OP(total_wait_time, uint64_t, "total_wait_ns") \ - OP(max_wait_time, uint64_t, "max_wait_ns") + OP(num_ops, uint64_t, "n_lock_ops", false, num_ops) \ + OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops) \ + OP(num_wait, uint64_t, "n_waiting", false, num_wait) \ + OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \ + OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \ + OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \ + OP(num_owner_switch, uint64_t, "n_owner_switch", false, num_owner_switch) \ + OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \ + OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \ + OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \ + OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time) #define MUTEX_PROF_UINT32_COUNTERS \ - OP(max_num_thds, uint32_t, "max_n_thds") + OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds) #define MUTEX_PROF_COUNTERS \ MUTEX_PROF_UINT64_COUNTERS \ MUTEX_PROF_UINT32_COUNTERS -#define OP(counter, type, human) mutex_counter_##counter, +#define OP(counter, type, human, derived, base_counter) mutex_counter_##counter, #define COUNTER_ENUM(counter_list, t) \ typedef enum { \ diff --git a/deps/jemalloc/include/jemalloc/internal/prof_externs.h b/deps/jemalloc/include/jemalloc/internal/prof_externs.h index 04348696f..094f3e170 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_externs.h @@ -14,6 +14,7 @@ extern bool opt_prof_gdump; /* High-water memory dumping. */ extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_leak; /* Dump leak summary at exit. */ extern bool opt_prof_accum; /* Report cumulative bytes. */ +extern bool opt_prof_log; /* Turn logging on at boot. */ extern char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -45,7 +46,8 @@ extern size_t lg_prof_sample; void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx); -void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); +void prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, + prof_tctx_t *tctx); void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt); prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); @@ -89,4 +91,15 @@ void prof_postfork_parent(tsdn_t *tsdn); void prof_postfork_child(tsdn_t *tsdn); void prof_sample_threshold_update(prof_tdata_t *tdata); +bool prof_log_start(tsdn_t *tsdn, const char *filename); +bool prof_log_stop(tsdn_t *tsdn); +#ifdef JEMALLOC_JET +size_t prof_log_bt_count(void); +size_t prof_log_alloc_count(void); +size_t prof_log_thr_count(void); +bool prof_log_is_logging(void); +bool prof_log_rep_check(void); +void prof_log_dummy_set(bool new_value); +#endif + #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h b/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h index a6efb4851..471d9853c 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h @@ -4,7 +4,8 @@ #include "jemalloc/internal/mutex.h" static inline bool -prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { +prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, + uint64_t accumbytes) { cassert(config_prof); bool overflow; @@ -42,7 +43,8 @@ prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { } static inline void -prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { +prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, + size_t usize) { cassert(config_prof); /* @@ -55,15 +57,15 @@ prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { #ifdef JEMALLOC_ATOMIC_U64 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); do { - a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - - usize) : 0; + a1 = (a0 >= SC_LARGE_MINCLASS - usize) + ? a0 - (SC_LARGE_MINCLASS - usize) : 0; } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); #else malloc_mutex_lock(tsdn, &prof_accum->mtx); a0 = prof_accum->accumbytes; - a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : - 0; + a1 = (a0 >= SC_LARGE_MINCLASS - usize) + ? a0 - (SC_LARGE_MINCLASS - usize) : 0; prof_accum->accumbytes = a1; malloc_mutex_unlock(tsdn, &prof_accum->mtx); #endif diff --git a/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h b/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h index 6ff465ad7..8ba8a1e1f 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PROF_INLINES_B_H #define JEMALLOC_INTERNAL_PROF_INLINES_B_H +#include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/sz.h" JEMALLOC_ALWAYS_INLINE bool @@ -61,13 +62,54 @@ prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { arena_prof_tctx_reset(tsdn, ptr, tctx); } +JEMALLOC_ALWAYS_INLINE nstime_t +prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { + cassert(config_prof); + assert(ptr != NULL); + + return arena_prof_alloc_time_get(tsdn, ptr, alloc_ctx); +} + +JEMALLOC_ALWAYS_INLINE void +prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, + nstime_t t) { + cassert(config_prof); + assert(ptr != NULL); + + arena_prof_alloc_time_set(tsdn, ptr, alloc_ctx, t); +} + +JEMALLOC_ALWAYS_INLINE bool +prof_sample_check(tsd_t *tsd, size_t usize, bool update) { + ssize_t check = update ? 0 : usize; + + int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); + if (update) { + bytes_until_sample -= usize; + if (tsd_nominal(tsd)) { + tsd_bytes_until_sample_set(tsd, bytes_until_sample); + } + } + if (likely(bytes_until_sample >= check)) { + return true; + } + + return false; +} + JEMALLOC_ALWAYS_INLINE bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, - prof_tdata_t **tdata_out) { + prof_tdata_t **tdata_out) { prof_tdata_t *tdata; cassert(config_prof); + /* Fastpath: no need to load tdata */ + if (likely(prof_sample_check(tsd, usize, update))) { + return true; + } + + bool booted = tsd_prof_tdata_get(tsd); tdata = prof_tdata_get(tsd, true); if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)) { tdata = NULL; @@ -81,21 +123,23 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, return true; } - if (likely(tdata->bytes_until_sample >= usize)) { - if (update) { - tdata->bytes_until_sample -= usize; - } + /* + * If this was the first creation of tdata, then + * prof_tdata_get() reset bytes_until_sample, so decrement and + * check it again + */ + if (!booted && prof_sample_check(tsd, usize, update)) { return true; - } else { - if (tsd_reentrancy_level_get(tsd) > 0) { - return true; - } - /* Compute new sample threshold. */ - if (update) { - prof_sample_threshold_update(tdata); - } - return !tdata->active; } + + if (tsd_reentrancy_level_get(tsd) > 0) { + return true; + } + /* Compute new sample threshold. */ + if (update) { + prof_sample_threshold_update(tdata); + } + return !tdata->active; } JEMALLOC_ALWAYS_INLINE prof_tctx_t * @@ -187,7 +231,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, * counters. */ if (unlikely(old_sampled)) { - prof_free_sampled_object(tsd, old_usize, old_tctx); + prof_free_sampled_object(tsd, ptr, old_usize, old_tctx); } } @@ -199,7 +243,7 @@ prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) { assert(usize == isalloc(tsd_tsdn(tsd), ptr)); if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) { - prof_free_sampled_object(tsd, usize, tctx); + prof_free_sampled_object(tsd, ptr, usize, tctx); } } diff --git a/deps/jemalloc/include/jemalloc/internal/prof_structs.h b/deps/jemalloc/include/jemalloc/internal/prof_structs.h index 0d58ae100..34ed4822b 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_structs.h @@ -169,7 +169,6 @@ struct prof_tdata_s { /* Sampling state. */ uint64_t prng_state; - uint64_t bytes_until_sample; /* State used to avoid dumping while operating on prof internals. */ bool enq; diff --git a/deps/jemalloc/include/jemalloc/internal/quantum.h b/deps/jemalloc/include/jemalloc/internal/quantum.h new file mode 100644 index 000000000..821086e99 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/quantum.h @@ -0,0 +1,77 @@ +#ifndef JEMALLOC_INTERNAL_QUANTUM_H +#define JEMALLOC_INTERNAL_QUANTUM_H + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#ifndef LG_QUANTUM +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# ifdef __aarch64__ +# define LG_QUANTUM 4 +# endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif +# ifdef __m68k__ +# define LG_QUANTUM 3 +# endif +# ifdef __mips__ +# define LG_QUANTUM 3 +# endif +# ifdef __nios2__ +# define LG_QUANTUM 3 +# endif +# ifdef __or1k__ +# define LG_QUANTUM 3 +# endif +# ifdef __powerpc__ +# define LG_QUANTUM 4 +# endif +# if defined(__riscv) || defined(__riscv__) +# define LG_QUANTUM 4 +# endif +# ifdef __s390__ +# define LG_QUANTUM 4 +# endif +# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ + defined(__SH4_SINGLE_ONLY__)) +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifdef __le32__ +# define LG_QUANTUM 4 +# endif +# ifndef LG_QUANTUM +# error "Unknown minimum alignment for architecture; specify via " + "--with-lg-quantum" +# endif +#endif + +#define QUANTUM ((size_t)(1U << LG_QUANTUM)) +#define QUANTUM_MASK (QUANTUM - 1) + +/* Return the smallest quantum multiple that is >= a. */ +#define QUANTUM_CEILING(a) \ + (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) + +#endif /* JEMALLOC_INTERNAL_QUANTUM_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/rtree.h b/deps/jemalloc/include/jemalloc/internal/rtree.h index b59d33a80..16ccbebee 100644 --- a/deps/jemalloc/include/jemalloc/internal/rtree.h +++ b/deps/jemalloc/include/jemalloc/internal/rtree.h @@ -4,7 +4,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree_tsd.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/tsd.h" /* @@ -31,7 +31,7 @@ # error Unsupported number of significant virtual address bits #endif /* Use compact leaf representation if virtual address encoding allows. */ -#if RTREE_NHIB >= LG_CEIL_NSIZES +#if RTREE_NHIB >= LG_CEIL(SC_NSIZES) # define RTREE_LEAF_COMPACT #endif @@ -170,8 +170,8 @@ rtree_subkey(uintptr_t key, unsigned level) { */ # ifdef RTREE_LEAF_COMPACT JEMALLOC_ALWAYS_INLINE uintptr_t -rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool dependent) { +rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool dependent) { return (uintptr_t)atomic_load_p(&elm->le_bits, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); } @@ -208,7 +208,7 @@ rtree_leaf_elm_bits_slab_get(uintptr_t bits) { # endif JEMALLOC_ALWAYS_INLINE extent_t * -rtree_leaf_elm_extent_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -221,7 +221,7 @@ rtree_leaf_elm_extent_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } JEMALLOC_ALWAYS_INLINE szind_t -rtree_leaf_elm_szind_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -233,7 +233,7 @@ rtree_leaf_elm_szind_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } JEMALLOC_ALWAYS_INLINE bool -rtree_leaf_elm_slab_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -245,7 +245,7 @@ rtree_leaf_elm_slab_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_extent_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, extent_t *extent) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, true); @@ -259,9 +259,9 @@ rtree_leaf_elm_extent_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_szind_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind) { - assert(szind <= NSIZES); + assert(szind <= SC_NSIZES); #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, @@ -277,7 +277,7 @@ rtree_leaf_elm_szind_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_slab_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, @@ -292,8 +292,8 @@ rtree_leaf_elm_slab_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - extent_t *extent, szind_t szind, bool slab) { +rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, extent_t *extent, szind_t szind, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = ((uintptr_t)szind << LG_VADDR) | ((uintptr_t)extent & (((uintptr_t)0x1 << LG_VADDR) - 1)) | @@ -313,7 +313,7 @@ rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, static inline void rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind, bool slab) { - assert(!slab || szind < NBINS); + assert(!slab || szind < SC_NBINS); /* * The caller implicitly assures that it is the only writer to the szind @@ -429,7 +429,7 @@ rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, dependent); if (!dependent && elm == NULL) { - return NSIZES; + return SC_NSIZES; } return rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent); } @@ -452,6 +452,42 @@ rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, return false; } +/* + * Try to read szind_slab from the L1 cache. Returns true on a hit, + * and fills in r_szind and r_slab. Otherwise returns false. + * + * Key is allowed to be NULL in order to save an extra branch on the + * fastpath. returns false in this case. + */ +JEMALLOC_ALWAYS_INLINE bool +rtree_szind_slab_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, szind_t *r_szind, bool *r_slab) { + rtree_leaf_elm_t *elm; + + size_t slot = rtree_cache_direct_map(key); + uintptr_t leafkey = rtree_leafkey(key); + assert(leafkey != RTREE_LEAFKEY_INVALID); + + if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) { + rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; + assert(leaf != NULL); + uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); + elm = &leaf[subkey]; + +#ifdef RTREE_LEAF_COMPACT + uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, + elm, true); + *r_szind = rtree_leaf_elm_bits_szind_get(bits); + *r_slab = rtree_leaf_elm_bits_slab_get(bits); +#else + *r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, true); + *r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, true); +#endif + return true; + } else { + return false; + } +} JEMALLOC_ALWAYS_INLINE bool rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) { @@ -474,7 +510,7 @@ rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, static inline void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab) { - assert(!slab || szind < NBINS); + assert(!slab || szind < SC_NBINS); rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab); @@ -486,7 +522,7 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); assert(rtree_leaf_elm_extent_read(tsdn, rtree, elm, false) != NULL); - rtree_leaf_elm_write(tsdn, rtree, elm, NULL, NSIZES, false); + rtree_leaf_elm_write(tsdn, rtree, elm, NULL, SC_NSIZES, false); } #endif /* JEMALLOC_INTERNAL_RTREE_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h b/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h index 93a75173a..562e29297 100644 --- a/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h +++ b/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h @@ -26,7 +26,7 @@ * Zero initializer required for tsd initialization only. Proper initialization * done via rtree_ctx_data_init(). */ -#define RTREE_CTX_ZERO_INITIALIZER {{{0}}, {{0}}} +#define RTREE_CTX_ZERO_INITIALIZER {{{0, 0}}, {{0, 0}}} typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; diff --git a/deps/jemalloc/include/jemalloc/internal/safety_check.h b/deps/jemalloc/include/jemalloc/internal/safety_check.h new file mode 100644 index 000000000..53339ac12 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/safety_check.h @@ -0,0 +1,26 @@ +#ifndef JEMALLOC_INTERNAL_SAFETY_CHECK_H +#define JEMALLOC_INTERNAL_SAFETY_CHECK_H + +void safety_check_fail(const char *format, ...); +/* Can set to NULL for a default. */ +void safety_check_set_abort(void (*abort_fn)()); + +JEMALLOC_ALWAYS_INLINE void +safety_check_set_redzone(void *ptr, size_t usize, size_t bumped_usize) { + assert(usize < bumped_usize); + for (size_t i = usize; i < bumped_usize && i < usize + 32; ++i) { + *((unsigned char *)ptr + i) = 0xBC; + } +} + +JEMALLOC_ALWAYS_INLINE void +safety_check_verify_redzone(const void *ptr, size_t usize, size_t bumped_usize) +{ + for (size_t i = usize; i < bumped_usize && i < usize + 32; ++i) { + if (unlikely(*((unsigned char *)ptr + i) != 0xBC)) { + safety_check_fail("Use after free error\n"); + } + } +} + +#endif /*JEMALLOC_INTERNAL_SAFETY_CHECK_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/sc.h b/deps/jemalloc/include/jemalloc/internal/sc.h new file mode 100644 index 000000000..9a099d8b6 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/sc.h @@ -0,0 +1,333 @@ +#ifndef JEMALLOC_INTERNAL_SC_H +#define JEMALLOC_INTERNAL_SC_H + +#include "jemalloc/internal/jemalloc_internal_types.h" + +/* + * Size class computations: + * + * These are a little tricky; we'll first start by describing how things + * generally work, and then describe some of the details. + * + * Ignore the first few size classes for a moment. We can then split all the + * remaining size classes into groups. The size classes in a group are spaced + * such that they cover allocation request sizes in a power-of-2 range. The + * power of two is called the base of the group, and the size classes in it + * satisfy allocations in the half-open range (base, base * 2]. There are + * SC_NGROUP size classes in each group, equally spaced in the range, so that + * each one covers allocations for base / SC_NGROUP possible allocation sizes. + * We call that value (base / SC_NGROUP) the delta of the group. Each size class + * is delta larger than the one before it (including the initial size class in a + * group, which is delta larger than base, the largest size class in the + * previous group). + * To make the math all work out nicely, we require that SC_NGROUP is a power of + * two, and define it in terms of SC_LG_NGROUP. We'll often talk in terms of + * lg_base and lg_delta. For each of these groups then, we have that + * lg_delta == lg_base - SC_LG_NGROUP. + * The size classes in a group with a given lg_base and lg_delta (which, recall, + * can be computed from lg_base for these groups) are therefore: + * base + 1 * delta + * which covers allocations in (base, base + 1 * delta] + * base + 2 * delta + * which covers allocations in (base + 1 * delta, base + 2 * delta]. + * base + 3 * delta + * which covers allocations in (base + 2 * delta, base + 3 * delta]. + * ... + * base + SC_NGROUP * delta ( == 2 * base) + * which covers allocations in (base + (SC_NGROUP - 1) * delta, 2 * base]. + * (Note that currently SC_NGROUP is always 4, so the "..." is empty in + * practice.) + * Note that the last size class in the group is the next power of two (after + * base), so that we've set up the induction correctly for the next group's + * selection of delta. + * + * Now, let's start considering the first few size classes. Two extra constants + * come into play here: LG_QUANTUM and SC_LG_TINY_MIN. LG_QUANTUM ensures + * correct platform alignment; all objects of size (1 << LG_QUANTUM) or larger + * are at least (1 << LG_QUANTUM) aligned; this can be used to ensure that we + * never return improperly aligned memory, by making (1 << LG_QUANTUM) equal the + * highest required alignment of a platform. For allocation sizes smaller than + * (1 << LG_QUANTUM) though, we can be more relaxed (since we don't support + * platforms with types with alignment larger than their size). To allow such + * allocations (without wasting space unnecessarily), we introduce tiny size + * classes; one per power of two, up until we hit the quantum size. There are + * therefore LG_QUANTUM - SC_LG_TINY_MIN such size classes. + * + * Next, we have a size class of size (1 << LG_QUANTUM). This can't be the + * start of a group in the sense we described above (covering a power of two + * range) since, if we divided into it to pick a value of delta, we'd get a + * delta smaller than (1 << LG_QUANTUM) for sizes >= (1 << LG_QUANTUM), which + * is against the rules. + * + * The first base we can divide by SC_NGROUP while still being at least + * (1 << LG_QUANTUM) is SC_NGROUP * (1 << LG_QUANTUM). We can get there by + * having SC_NGROUP size classes, spaced (1 << LG_QUANTUM) apart. These size + * classes are: + * 1 * (1 << LG_QUANTUM) + * 2 * (1 << LG_QUANTUM) + * 3 * (1 << LG_QUANTUM) + * ... (although, as above, this "..." is empty in practice) + * SC_NGROUP * (1 << LG_QUANTUM). + * + * There are SC_NGROUP of these size classes, so we can regard it as a sort of + * pseudo-group, even though it spans multiple powers of 2, is divided + * differently, and both starts and ends on a power of 2 (as opposed to just + * ending). SC_NGROUP is itself a power of two, so the first group after the + * pseudo-group has the power-of-two base SC_NGROUP * (1 << LG_QUANTUM), for a + * lg_base of LG_QUANTUM + SC_LG_NGROUP. We can divide this base into SC_NGROUP + * sizes without violating our LG_QUANTUM requirements, so we can safely set + * lg_delta = lg_base - SC_LG_GROUP (== LG_QUANTUM). + * + * So, in order, the size classes are: + * + * Tiny size classes: + * - Count: LG_QUANTUM - SC_LG_TINY_MIN. + * - Sizes: + * 1 << SC_LG_TINY_MIN + * 1 << (SC_LG_TINY_MIN + 1) + * 1 << (SC_LG_TINY_MIN + 2) + * ... + * 1 << (LG_QUANTUM - 1) + * + * Initial pseudo-group: + * - Count: SC_NGROUP + * - Sizes: + * 1 * (1 << LG_QUANTUM) + * 2 * (1 << LG_QUANTUM) + * 3 * (1 << LG_QUANTUM) + * ... + * SC_NGROUP * (1 << LG_QUANTUM) + * + * Regular group 0: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * Regular group 1: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + 1 and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * ... + * + * Regular group N: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + N and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * + * Representation of metadata: + * To make the math easy, we'll mostly work in lg quantities. We record lg_base, + * lg_delta, and ndelta (i.e. number of deltas above the base) on a + * per-size-class basis, and maintain the invariant that, across all size + * classes, size == (1 << lg_base) + ndelta * (1 << lg_delta). + * + * For regular groups (i.e. those with lg_base >= LG_QUANTUM + SC_LG_NGROUP), + * lg_delta is lg_base - SC_LG_NGROUP, and ndelta goes from 1 to SC_NGROUP. + * + * For the initial tiny size classes (if any), lg_base is lg(size class size). + * lg_delta is lg_base for the first size class, and lg_base - 1 for all + * subsequent ones. ndelta is always 0. + * + * For the pseudo-group, if there are no tiny size classes, then we set + * lg_base == LG_QUANTUM, lg_delta == LG_QUANTUM, and have ndelta range from 0 + * to SC_NGROUP - 1. (Note that delta == base, so base + (SC_NGROUP - 1) * delta + * is just SC_NGROUP * base, or (1 << (SC_LG_NGROUP + LG_QUANTUM)), so we do + * indeed get a power of two that way). If there *are* tiny size classes, then + * the first size class needs to have lg_delta relative to the largest tiny size + * class. We therefore set lg_base == LG_QUANTUM - 1, + * lg_delta == LG_QUANTUM - 1, and ndelta == 1, keeping the rest of the + * pseudo-group the same. + * + * + * Other terminology: + * "Small" size classes mean those that are allocated out of bins, which is the + * same as those that are slab allocated. + * "Large" size classes are those that are not small. The cutoff for counting as + * large is page size * group size. + */ + +/* + * Size class N + (1 << SC_LG_NGROUP) twice the size of size class N. + */ +#define SC_LG_NGROUP 2 +#define SC_LG_TINY_MIN 3 + +#if SC_LG_TINY_MIN == 0 +/* The div module doesn't support division by 1, which this would require. */ +#error "Unsupported LG_TINY_MIN" +#endif + +/* + * The definitions below are all determined by the above settings and system + * characteristics. + */ +#define SC_NGROUP (1ULL << SC_LG_NGROUP) +#define SC_PTR_BITS ((1ULL << LG_SIZEOF_PTR) * 8) +#define SC_NTINY (LG_QUANTUM - SC_LG_TINY_MIN) +#define SC_LG_TINY_MAXCLASS (LG_QUANTUM > SC_LG_TINY_MIN ? LG_QUANTUM - 1 : -1) +#define SC_NPSEUDO SC_NGROUP +#define SC_LG_FIRST_REGULAR_BASE (LG_QUANTUM + SC_LG_NGROUP) +/* + * We cap allocations to be less than 2 ** (ptr_bits - 1), so the highest base + * we need is 2 ** (ptr_bits - 2). (This also means that the last group is 1 + * size class shorter than the others). + * We could probably save some space in arenas by capping this at LG_VADDR size. + */ +#define SC_LG_BASE_MAX (SC_PTR_BITS - 2) +#define SC_NREGULAR (SC_NGROUP * \ + (SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1) +#define SC_NSIZES (SC_NTINY + SC_NPSEUDO + SC_NREGULAR) + +/* The number of size classes that are a multiple of the page size. */ +#define SC_NPSIZES ( \ + /* Start with all the size classes. */ \ + SC_NSIZES \ + /* Subtract out those groups with too small a base. */ \ + - (LG_PAGE - 1 - SC_LG_FIRST_REGULAR_BASE) * SC_NGROUP \ + /* And the pseudo-group. */ \ + - SC_NPSEUDO \ + /* And the tiny group. */ \ + - SC_NTINY \ + /* Sizes where ndelta*delta is not a multiple of the page size. */ \ + - (SC_LG_NGROUP * SC_NGROUP)) +/* + * Note that the last line is computed as the sum of the second column in the + * following table: + * lg(base) | count of sizes to exclude + * ------------------------------|----------------------------- + * LG_PAGE - 1 | SC_NGROUP - 1 + * LG_PAGE | SC_NGROUP - 1 + * LG_PAGE + 1 | SC_NGROUP - 2 + * LG_PAGE + 2 | SC_NGROUP - 4 + * ... | ... + * LG_PAGE + (SC_LG_NGROUP - 1) | SC_NGROUP - (SC_NGROUP / 2) + */ + +/* + * We declare a size class is binnable if size < page size * group. Or, in other + * words, lg(size) < lg(page size) + lg(group size). + */ +#define SC_NBINS ( \ + /* Sub-regular size classes. */ \ + SC_NTINY + SC_NPSEUDO \ + /* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \ + + SC_NGROUP * (LG_PAGE + SC_LG_NGROUP - SC_LG_FIRST_REGULAR_BASE) \ + /* Last SC of the last group hits the bound exactly; exclude it. */ \ + - 1) + +/* + * The size2index_tab lookup table uses uint8_t to encode each bin index, so we + * cannot support more than 256 small size classes. + */ +#if (SC_NBINS > 256) +# error "Too many small size classes" +#endif + +/* The largest size class in the lookup table. */ +#define SC_LOOKUP_MAXCLASS ((size_t)1 << 12) + +/* Internal, only used for the definition of SC_SMALL_MAXCLASS. */ +#define SC_SMALL_MAX_BASE ((size_t)1 << (LG_PAGE + SC_LG_NGROUP - 1)) +#define SC_SMALL_MAX_DELTA ((size_t)1 << (LG_PAGE - 1)) + +/* The largest size class allocated out of a slab. */ +#define SC_SMALL_MAXCLASS (SC_SMALL_MAX_BASE \ + + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) + +/* The smallest size class not allocated out of a slab. */ +#define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP)) +#define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP) + +/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */ +#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2)) +#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP)) + +/* The largest size class supported. */ +#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA) + +typedef struct sc_s sc_t; +struct sc_s { + /* Size class index, or -1 if not a valid size class. */ + int index; + /* Lg group base size (no deltas added). */ + int lg_base; + /* Lg delta to previous size class. */ + int lg_delta; + /* Delta multiplier. size == 1<data) / sizeof(size_t)]; \ + buf[sizeof(buf) / sizeof(size_t) - 1] = 0; \ + memcpy(buf, src, sizeof(type)); \ + size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED); \ + atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED); \ + atomic_fence(ATOMIC_RELEASE); \ + for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ + atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED); \ + } \ + atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE); \ +} \ + \ +/* Returns whether or not the read was consistent. */ \ +static inline bool \ +seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) { \ + size_t buf[sizeof(src->data) / sizeof(size_t)]; \ + size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE); \ + if (seq1 % 2 != 0) { \ + return false; \ + } \ + for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ + buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED); \ + } \ + atomic_fence(ATOMIC_ACQUIRE); \ + size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED); \ + if (seq1 != seq2) { \ + return false; \ + } \ + memcpy(dst, buf, sizeof(type)); \ + return true; \ +} + +#endif /* JEMALLOC_INTERNAL_SEQ_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/size_classes.sh b/deps/jemalloc/include/jemalloc/internal/size_classes.sh deleted file mode 100755 index 998994d09..000000000 --- a/deps/jemalloc/include/jemalloc/internal/size_classes.sh +++ /dev/null @@ -1,361 +0,0 @@ -#!/bin/sh -# -# Usage: size_classes.sh - -# The following limits are chosen such that they cover all supported platforms. - -# Pointer sizes. -lg_zarr="2 3" - -# Quanta. -lg_qarr=$1 - -# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)]. -lg_tmin=$2 - -# Maximum lookup size. -lg_kmax=12 - -# Page sizes. -lg_parr=`echo $3 | tr ',' ' '` - -# Size class group size (number of size classes for each size doubling). -lg_g=$4 - -pow2() { - e=$1 - pow2_result=1 - while [ ${e} -gt 0 ] ; do - pow2_result=$((${pow2_result} + ${pow2_result})) - e=$((${e} - 1)) - done -} - -lg() { - x=$1 - lg_result=0 - while [ ${x} -gt 1 ] ; do - lg_result=$((${lg_result} + 1)) - x=$((${x} / 2)) - done -} - -lg_ceil() { - y=$1 - lg ${y}; lg_floor=${lg_result} - pow2 ${lg_floor}; pow2_floor=${pow2_result} - if [ ${pow2_floor} -lt ${y} ] ; then - lg_ceil_result=$((${lg_floor} + 1)) - else - lg_ceil_result=${lg_floor} - fi -} - -reg_size_compute() { - lg_grp=$1 - lg_delta=$2 - ndelta=$3 - - pow2 ${lg_grp}; grp=${pow2_result} - pow2 ${lg_delta}; delta=${pow2_result} - reg_size=$((${grp} + ${delta}*${ndelta})) -} - -slab_size() { - lg_p=$1 - lg_grp=$2 - lg_delta=$3 - ndelta=$4 - - pow2 ${lg_p}; p=${pow2_result} - reg_size_compute ${lg_grp} ${lg_delta} ${ndelta} - - # Compute smallest slab size that is an integer multiple of reg_size. - try_slab_size=${p} - try_nregs=$((${try_slab_size} / ${reg_size})) - perfect=0 - while [ ${perfect} -eq 0 ] ; do - perfect_slab_size=${try_slab_size} - perfect_nregs=${try_nregs} - - try_slab_size=$((${try_slab_size} + ${p})) - try_nregs=$((${try_slab_size} / ${reg_size})) - if [ ${perfect_slab_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then - perfect=1 - fi - done - - slab_size_pgs=$((${perfect_slab_size} / ${p})) -} - -size_class() { - index=$1 - lg_grp=$2 - lg_delta=$3 - ndelta=$4 - lg_p=$5 - lg_kmax=$6 - - if [ ${lg_delta} -ge ${lg_p} ] ; then - psz="yes" - else - pow2 ${lg_p}; p=${pow2_result} - pow2 ${lg_grp}; grp=${pow2_result} - pow2 ${lg_delta}; delta=${pow2_result} - sz=$((${grp} + ${delta} * ${ndelta})) - npgs=$((${sz} / ${p})) - if [ ${sz} -eq $((${npgs} * ${p})) ] ; then - psz="yes" - else - psz="no" - fi - fi - - lg ${ndelta}; lg_ndelta=${lg_result}; pow2 ${lg_ndelta} - if [ ${pow2_result} -lt ${ndelta} ] ; then - rem="yes" - else - rem="no" - fi - - lg_size=${lg_grp} - if [ $((${lg_delta} + ${lg_ndelta})) -eq ${lg_grp} ] ; then - lg_size=$((${lg_grp} + 1)) - else - lg_size=${lg_grp} - rem="yes" - fi - - if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then - bin="yes" - slab_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${slab_size_pgs} - else - bin="no" - pgs=0 - fi - if [ ${lg_size} -lt ${lg_kmax} \ - -o ${lg_size} -eq ${lg_kmax} -a ${rem} = "no" ] ; then - lg_delta_lookup=${lg_delta} - else - lg_delta_lookup="no" - fi - printf ' SC(%3d, %6d, %8d, %6d, %3s, %3s, %3d, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${psz} ${bin} ${pgs} ${lg_delta_lookup} - # Defined upon return: - # - psz ("yes" or "no") - # - bin ("yes" or "no") - # - pgs - # - lg_delta_lookup (${lg_delta} or "no") -} - -sep_line() { - echo " \\" -} - -size_classes() { - lg_z=$1 - lg_q=$2 - lg_t=$3 - lg_p=$4 - lg_g=$5 - - pow2 $((${lg_z} + 3)); ptr_bits=${pow2_result} - pow2 ${lg_g}; g=${pow2_result} - - echo "#define SIZE_CLASSES \\" - echo " /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \\" - - ntbins=0 - nlbins=0 - lg_tiny_maxclass='"NA"' - nbins=0 - npsizes=0 - - # Tiny size classes. - ndelta=0 - index=0 - lg_grp=${lg_t} - lg_delta=${lg_grp} - while [ ${lg_grp} -lt ${lg_q} ] ; do - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - if [ ${lg_delta_lookup} != "no" ] ; then - nlbins=$((${index} + 1)) - fi - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - if [ ${bin} != "no" ] ; then - nbins=$((${index} + 1)) - fi - ntbins=$((${ntbins} + 1)) - lg_tiny_maxclass=${lg_grp} # Final written value is correct. - index=$((${index} + 1)) - lg_delta=${lg_grp} - lg_grp=$((${lg_grp} + 1)) - done - - # First non-tiny group. - if [ ${ntbins} -gt 0 ] ; then - sep_line - # The first size class has an unusual encoding, because the size has to be - # split between grp and delta*ndelta. - lg_grp=$((${lg_grp} - 1)) - ndelta=1 - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - index=$((${index} + 1)) - lg_grp=$((${lg_grp} + 1)) - lg_delta=$((${lg_delta} + 1)) - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - fi - while [ ${ndelta} -lt ${g} ] ; do - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - index=$((${index} + 1)) - ndelta=$((${ndelta} + 1)) - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - done - - # All remaining groups. - lg_grp=$((${lg_grp} + ${lg_g})) - while [ ${lg_grp} -lt $((${ptr_bits} - 1)) ] ; do - sep_line - ndelta=1 - if [ ${lg_grp} -eq $((${ptr_bits} - 2)) ] ; then - ndelta_limit=$((${g} - 1)) - else - ndelta_limit=${g} - fi - while [ ${ndelta} -le ${ndelta_limit} ] ; do - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - if [ ${lg_delta_lookup} != "no" ] ; then - nlbins=$((${index} + 1)) - # Final written value is correct: - lookup_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - fi - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - if [ ${bin} != "no" ] ; then - nbins=$((${index} + 1)) - # Final written value is correct: - small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - if [ ${lg_g} -gt 0 ] ; then - lg_large_minclass=$((${lg_grp} + 1)) - else - lg_large_minclass=$((${lg_grp} + 2)) - fi - fi - # Final written value is correct: - large_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - index=$((${index} + 1)) - ndelta=$((${ndelta} + 1)) - done - lg_grp=$((${lg_grp} + 1)) - lg_delta=$((${lg_delta} + 1)) - done - echo - nsizes=${index} - lg_ceil ${nsizes}; lg_ceil_nsizes=${lg_ceil_result} - - # Defined upon completion: - # - ntbins - # - nlbins - # - nbins - # - nsizes - # - lg_ceil_nsizes - # - npsizes - # - lg_tiny_maxclass - # - lookup_maxclass - # - small_maxclass - # - lg_large_minclass - # - large_maxclass -} - -cat < 256) -# error "Too many small size classes" -#endif - -#endif /* JEMALLOC_INTERNAL_SIZE_CLASSES_H */ -EOF diff --git a/deps/jemalloc/include/jemalloc/internal/stats.h b/deps/jemalloc/include/jemalloc/internal/stats.h index 852e34269..3b9e0eac1 100644 --- a/deps/jemalloc/include/jemalloc/internal/stats.h +++ b/deps/jemalloc/include/jemalloc/internal/stats.h @@ -10,7 +10,8 @@ OPTION('a', unmerged, config_stats, false) \ OPTION('b', bins, true, false) \ OPTION('l', large, true, false) \ - OPTION('x', mutex, true, false) + OPTION('x', mutex, true, false) \ + OPTION('e', extents, true, false) enum { #define OPTION(o, v, d, s) stats_print_option_num_##v, diff --git a/deps/jemalloc/include/jemalloc/internal/sz.h b/deps/jemalloc/include/jemalloc/internal/sz.h index 979462898..68e558abf 100644 --- a/deps/jemalloc/include/jemalloc/internal/sz.h +++ b/deps/jemalloc/include/jemalloc/internal/sz.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/util.h" /* @@ -26,18 +26,18 @@ * sz_pind2sz_tab encodes the same information as could be computed by * sz_pind2sz_compute(). */ -extern size_t const sz_pind2sz_tab[NPSIZES+1]; +extern size_t sz_pind2sz_tab[SC_NPSIZES + 1]; /* * sz_index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by sz_index2size_compute(). */ -extern size_t const sz_index2size_tab[NSIZES]; +extern size_t sz_index2size_tab[SC_NSIZES]; /* * sz_size2index_tab is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, * and all accesses are via sz_size2index(). */ -extern uint8_t const sz_size2index_tab[]; +extern uint8_t sz_size2index_tab[]; static const size_t sz_large_pad = #ifdef JEMALLOC_CACHE_OBLIVIOUS @@ -47,49 +47,47 @@ static const size_t sz_large_pad = #endif ; +extern void sz_boot(const sc_data_t *sc_data); + JEMALLOC_ALWAYS_INLINE pszind_t sz_psz2ind(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return NPSIZES; + if (unlikely(psz > SC_LARGE_MAXCLASS)) { + return SC_NPSIZES; } - { - pszind_t x = lg_floor((psz<<1)-1); - pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - - (LG_SIZE_CLASS_GROUP + LG_PAGE); - pszind_t grp = shift << LG_SIZE_CLASS_GROUP; + pszind_t x = lg_floor((psz<<1)-1); + pszind_t shift = (x < SC_LG_NGROUP + LG_PAGE) ? + 0 : x - (SC_LG_NGROUP + LG_PAGE); + pszind_t grp = shift << SC_LG_NGROUP; - pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + pszind_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? + LG_PAGE : x - SC_LG_NGROUP - 1; - size_t delta_inverse_mask = ZU(-1) << lg_delta; - pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + size_t delta_inverse_mask = ZU(-1) << lg_delta; + pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << SC_LG_NGROUP) - 1); - pszind_t ind = grp + mod; - return ind; - } + pszind_t ind = grp + mod; + return ind; } static inline size_t sz_pind2sz_compute(pszind_t pind) { - if (unlikely(pind == NPSIZES)) { - return LARGE_MAXCLASS + PAGE; + if (unlikely(pind == SC_NPSIZES)) { + return SC_LARGE_MAXCLASS + PAGE; } - { - size_t grp = pind >> LG_SIZE_CLASS_GROUP; - size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + size_t grp = pind >> SC_LG_NGROUP; + size_t mod = pind & ((ZU(1) << SC_LG_NGROUP) - 1); - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_PAGE + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP-1))) << grp) + & grp_size_mask; - size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_PAGE-1); - size_t mod_size = (mod+1) << lg_delta; + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_PAGE-1); + size_t mod_size = (mod+1) << lg_delta; - size_t sz = grp_size + mod_size; - return sz; - } + size_t sz = grp_size + mod_size; + return sz; } static inline size_t @@ -101,70 +99,70 @@ sz_pind2sz_lookup(pszind_t pind) { static inline size_t sz_pind2sz(pszind_t pind) { - assert(pind < NPSIZES+1); + assert(pind < SC_NPSIZES + 1); return sz_pind2sz_lookup(pind); } static inline size_t sz_psz2u(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return LARGE_MAXCLASS + PAGE; - } - { - size_t x = lg_floor((psz<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (psz + delta_mask) & ~delta_mask; - return usize; + if (unlikely(psz > SC_LARGE_MAXCLASS)) { + return SC_LARGE_MAXCLASS + PAGE; } + size_t x = lg_floor((psz<<1)-1); + size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? + LG_PAGE : x - SC_LG_NGROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (psz + delta_mask) & ~delta_mask; + return usize; } static inline szind_t sz_size2index_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { - return NSIZES; + if (unlikely(size > SC_LARGE_MAXCLASS)) { + return SC_NSIZES; } -#if (NTBINS != 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + + if (size == 0) { + return 0; + } +#if (SC_NTINY != 0) + if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { + szind_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; szind_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); } #endif { szind_t x = lg_floor((size<<1)-1); - szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : - x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); - szind_t grp = shift << LG_SIZE_CLASS_GROUP; + szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM) ? 0 : + x - (SC_LG_NGROUP + LG_QUANTUM); + szind_t grp = shift << SC_LG_NGROUP; - szind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + szind_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - SC_LG_NGROUP - 1; size_t delta_inverse_mask = ZU(-1) << lg_delta; szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + ((ZU(1) << SC_LG_NGROUP) - 1); - szind_t index = NTBINS + grp + mod; + szind_t index = SC_NTINY + grp + mod; return index; } } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index_lookup(size_t size) { - assert(size <= LOOKUP_MAXCLASS); - { - szind_t ret = (sz_size2index_tab[(size-1) >> LG_TINY_MIN]); - assert(ret == sz_size2index_compute(size)); - return ret; - } + assert(size <= SC_LOOKUP_MAXCLASS); + szind_t ret = (sz_size2index_tab[(size + (ZU(1) << SC_LG_TINY_MIN) - 1) + >> SC_LG_TINY_MIN]); + assert(ret == sz_size2index_compute(size)); + return ret; } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { + if (likely(size <= SC_LOOKUP_MAXCLASS)) { return sz_size2index_lookup(size); } return sz_size2index_compute(size); @@ -172,20 +170,20 @@ sz_size2index(size_t size) { static inline size_t sz_index2size_compute(szind_t index) { -#if (NTBINS > 0) - if (index < NTBINS) { - return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); +#if (SC_NTINY > 0) + if (index < SC_NTINY) { + return (ZU(1) << (SC_LG_TINY_MAXCLASS - SC_NTINY + 1 + index)); } #endif { - size_t reduced_index = index - NTBINS; - size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; - size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - + size_t reduced_index = index - SC_NTINY; + size_t grp = reduced_index >> SC_LG_NGROUP; + size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - 1); size_t grp_size_mask = ~((!!grp)-1); size_t grp_size = ((ZU(1) << (LG_QUANTUM + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + (SC_LG_NGROUP-1))) << grp) & grp_size_mask; size_t shift = (grp == 0) ? 1 : grp; size_t lg_delta = shift + (LG_QUANTUM-1); @@ -205,18 +203,22 @@ sz_index2size_lookup(szind_t index) { JEMALLOC_ALWAYS_INLINE size_t sz_index2size(szind_t index) { - assert(index < NSIZES); + assert(index < SC_NSIZES); return sz_index2size_lookup(index); } JEMALLOC_ALWAYS_INLINE size_t sz_s2u_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { + if (unlikely(size > SC_LARGE_MAXCLASS)) { return 0; } -#if (NTBINS > 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + + if (size == 0) { + size++; + } +#if (SC_NTINY > 0) + if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { + size_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : (ZU(1) << lg_ceil)); @@ -224,8 +226,8 @@ sz_s2u_compute(size_t size) { #endif { size_t x = lg_floor((size<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - SC_LG_NGROUP - 1; size_t delta = ZU(1) << lg_delta; size_t delta_mask = delta - 1; size_t usize = (size + delta_mask) & ~delta_mask; @@ -247,8 +249,7 @@ sz_s2u_lookup(size_t size) { */ JEMALLOC_ALWAYS_INLINE size_t sz_s2u(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { + if (likely(size <= SC_LOOKUP_MAXCLASS)) { return sz_s2u_lookup(size); } return sz_s2u_compute(size); @@ -265,7 +266,7 @@ sz_sa2u(size_t size, size_t alignment) { assert(alignment != 0 && ((alignment - 1) & alignment) == 0); /* Try for a small size class. */ - if (size <= SMALL_MAXCLASS && alignment < PAGE) { + if (size <= SC_SMALL_MAXCLASS && alignment < PAGE) { /* * Round size up to the nearest multiple of alignment. * @@ -281,20 +282,20 @@ sz_sa2u(size_t size, size_t alignment) { * 192 | 11000000 | 64 */ usize = sz_s2u(ALIGNMENT_CEILING(size, alignment)); - if (usize < LARGE_MINCLASS) { + if (usize < SC_LARGE_MINCLASS) { return usize; } } /* Large size class. Beware of overflow. */ - if (unlikely(alignment > LARGE_MAXCLASS)) { + if (unlikely(alignment > SC_LARGE_MAXCLASS)) { return 0; } /* Make sure result is a large size class. */ - if (size <= LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (size <= SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } else { usize = sz_s2u(size); if (usize < size) { diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_externs.h b/deps/jemalloc/include/jemalloc/internal/tcache_externs.h index 790367bd4..d63eafde8 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_externs.h @@ -1,15 +1,13 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H #define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H -#include "jemalloc/internal/size_classes.h" - extern bool opt_tcache; extern ssize_t opt_lg_tcache_max; extern cache_bin_info_t *tcache_bin_info; /* - * Number of tcache bins. There are NBINS small-object bins, plus 0 or more + * Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more * large-object bins. */ extern unsigned nhbins; diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h b/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h index 0f6ab8cb5..5eca20e89 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/util.h" @@ -40,13 +40,13 @@ tcache_event(tsd_t *tsd, tcache_t *tcache) { JEMALLOC_ALWAYS_INLINE void * tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - UNUSED size_t size, szind_t binind, bool zero, bool slow_path) { + size_t size, szind_t binind, bool zero, bool slow_path) { void *ret; cache_bin_t *bin; bool tcache_success; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - assert(binind < NBINS); + assert(binind < SC_NBINS); bin = tcache_small_bin_get(tcache, binind); ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); @@ -107,7 +107,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, cache_bin_t *bin; bool tcache_success; - assert(binind >= NBINS &&binind < nhbins); + assert(binind >= SC_NBINS &&binind < nhbins); bin = tcache_large_bin_get(tcache, binind); ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); @@ -166,7 +166,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin; cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) + <= SC_SMALL_MAXCLASS); if (slow_path && config_fill && unlikely(opt_junk_free)) { arena_dalloc_junk_small(ptr, &bin_infos[binind]); @@ -174,13 +175,12 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bin = tcache_small_bin_get(tcache, binind); bin_info = &tcache_bin_info[binind]; - if (unlikely(bin->ncached == bin_info->ncached_max)) { + if (unlikely(!cache_bin_dalloc_easy(bin, bin_info, ptr))) { tcache_bin_flush_small(tsd, tcache, bin, binind, (bin_info->ncached_max >> 1)); + bool ret = cache_bin_dalloc_easy(bin, bin_info, ptr); + assert(ret); } - assert(bin->ncached < bin_info->ncached_max); - bin->ncached++; - *(bin->avail - bin->ncached) = ptr; tcache_event(tsd, tcache); } @@ -191,7 +191,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin; cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) + > SC_SMALL_MAXCLASS); assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass); if (slow_path && config_fill && unlikely(opt_junk_free)) { @@ -215,6 +216,9 @@ JEMALLOC_ALWAYS_INLINE tcache_t * tcaches_get(tsd_t *tsd, unsigned ind) { tcaches_t *elm = &tcaches[ind]; if (unlikely(elm->tcache == NULL)) { + malloc_printf(": invalid tcache id (%u).\n", ind); + abort(); + } else if (unlikely(elm->tcache == TCACHES_ELM_NEED_REINIT)) { elm->tcache = tcache_create_explicit(tsd); } return elm->tcache; diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_structs.h b/deps/jemalloc/include/jemalloc/internal/tcache_structs.h index 07b738705..172ef9040 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_structs.h @@ -1,10 +1,14 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H #define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H -#include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/cache_bin.h" +#include "jemalloc/internal/ql.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/ticker.h" +#include "jemalloc/internal/tsd_types.h" + +/* Various uses of this struct need it to be a named type. */ +typedef ql_elm(tsd_t) tsd_link_t; struct tcache_s { /* @@ -21,7 +25,7 @@ struct tcache_s { * During tcache initialization, the avail pointer in each element of * tbins is initialized to point to the proper offset within this array. */ - cache_bin_t bins_small[NBINS]; + cache_bin_t bins_small[SC_NBINS]; /* * This data is less hot; we can be a little less careful with our @@ -29,6 +33,11 @@ struct tcache_s { */ /* Lets us track all the tcaches in an arena. */ ql_elm(tcache_t) link; + + /* Logically scoped to tsd, but put here for cache layout reasons. */ + ql_elm(tsd_t) tsd_link; + bool in_hook; + /* * The descriptor lets the arena find our cache bins without seeing the * tcache definition. This enables arenas to aggregate stats across @@ -41,13 +50,13 @@ struct tcache_s { /* Next bin to GC. */ szind_t next_gc_bin; /* For small bins, fill (ncached_max >> lg_fill_div). */ - uint8_t lg_fill_div[NBINS]; + uint8_t lg_fill_div[SC_NBINS]; /* * We put the cache bins for large size classes at the end of the * struct, since some of them might not get used. This might end up * letting us avoid touching an extra page if we don't have to. */ - cache_bin_t bins_large[NSIZES-NBINS]; + cache_bin_t bins_large[SC_NSIZES-SC_NBINS]; }; /* Linkage for list of available (previously used) explicit tcache IDs. */ diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_types.h b/deps/jemalloc/include/jemalloc/internal/tcache_types.h index e49bc9d79..dce69382e 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_types.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_types.h @@ -1,7 +1,7 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H #define JEMALLOC_INTERNAL_TCACHE_TYPES_H -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef struct tcache_s tcache_t; typedef struct tcaches_s tcaches_t; @@ -45,7 +45,7 @@ typedef struct tcaches_s tcaches_t; /* Number of tcache allocation/deallocation events between incremental GCs. */ #define TCACHE_GC_INCR \ - ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) + ((TCACHE_GC_SWEEP / SC_NBINS) + ((TCACHE_GC_SWEEP / SC_NBINS == 0) ? 0 : 1)) /* Used in TSD static initializer only. Real init in tcache_data_init(). */ #define TCACHE_ZERO_INITIALIZER {0} @@ -53,4 +53,7 @@ typedef struct tcaches_s tcaches_t; /* Used in TSD static initializer only. Will be initialized to opt_tcache. */ #define TCACHE_ENABLED_ZERO_INITIALIZER false +/* Used for explicit tcache only. Means flushed but not destroyed. */ +#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1) + #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/test_hooks.h b/deps/jemalloc/include/jemalloc/internal/test_hooks.h new file mode 100644 index 000000000..a6351e59a --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/test_hooks.h @@ -0,0 +1,19 @@ +#ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H +#define JEMALLOC_INTERNAL_TEST_HOOKS_H + +extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(); +extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); + +#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) + +#define open JEMALLOC_HOOK(open, test_hooks_libc_hook) +#define read JEMALLOC_HOOK(read, test_hooks_libc_hook) +#define write JEMALLOC_HOOK(write, test_hooks_libc_hook) +#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook) +#define close JEMALLOC_HOOK(close, test_hooks_libc_hook) +#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook) +#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook) +/* Note that this is undef'd and re-define'd in src/prof.c. */ +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) + +#endif /* JEMALLOC_INTERNAL_TEST_HOOKS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/ticker.h b/deps/jemalloc/include/jemalloc/internal/ticker.h index 4b3604708..52d0db4c8 100644 --- a/deps/jemalloc/include/jemalloc/internal/ticker.h +++ b/deps/jemalloc/include/jemalloc/internal/ticker.h @@ -75,4 +75,17 @@ ticker_tick(ticker_t *ticker) { return ticker_ticks(ticker, 1); } +/* + * Try to tick. If ticker would fire, return true, but rely on + * slowpath to reset ticker. + */ +static inline bool +ticker_trytick(ticker_t *ticker) { + --ticker->tick; + if (unlikely(ticker->tick < 0)) { + return true; + } + return false; +} + #endif /* JEMALLOC_INTERNAL_TICKER_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/tsd.h b/deps/jemalloc/include/jemalloc/internal/tsd.h index 0b9841aa7..9ba260045 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd.h @@ -3,6 +3,7 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/prof_types.h" #include "jemalloc/internal/ql.h" @@ -68,17 +69,19 @@ typedef void (*test_callback_t)(int *); O(offset_state, uint64_t, uint64_t) \ O(thread_allocated, uint64_t, uint64_t) \ O(thread_deallocated, uint64_t, uint64_t) \ + O(bytes_until_sample, int64_t, int64_t) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \ O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ + O(binshards, tsd_binshards_t, tsd_binshards_t)\ O(tcache, tcache_t, tcache_t) \ O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ MALLOC_TEST_TSD #define TSD_INITIALIZER { \ - tsd_state_uninitialized, \ + ATOMIC_INIT(tsd_state_uninitialized), \ TCACHE_ENABLED_ZERO_INITIALIZER, \ false, \ 0, \ @@ -86,29 +89,97 @@ typedef void (*test_callback_t)(int *); 0, \ 0, \ 0, \ + 0, \ NULL, \ RTREE_CTX_ZERO_INITIALIZER, \ NULL, \ NULL, \ NULL, \ + TSD_BINSHARDS_ZERO_INITIALIZER, \ TCACHE_ZERO_INITIALIZER, \ WITNESS_TSD_INITIALIZER \ MALLOC_TEST_TSD_INITIALIZER \ } +void *malloc_tsd_malloc(size_t size); +void malloc_tsd_dalloc(void *wrapper); +void malloc_tsd_cleanup_register(bool (*f)(void)); +tsd_t *malloc_tsd_boot0(void); +void malloc_tsd_boot1(void); +void tsd_cleanup(void *arg); +tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); +void tsd_state_set(tsd_t *tsd, uint8_t new_state); +void tsd_slow_update(tsd_t *tsd); +void tsd_prefork(tsd_t *tsd); +void tsd_postfork_parent(tsd_t *tsd); +void tsd_postfork_child(tsd_t *tsd); + +/* + * Call ..._inc when your module wants to take all threads down the slow paths, + * and ..._dec when it no longer needs to. + */ +void tsd_global_slow_inc(tsdn_t *tsdn); +void tsd_global_slow_dec(tsdn_t *tsdn); +bool tsd_global_slow(); + enum { - tsd_state_nominal = 0, /* Common case --> jnz. */ - tsd_state_nominal_slow = 1, /* Initialized but on slow path. */ - /* the above 2 nominal states should be lower values. */ - tsd_state_nominal_max = 1, /* used for comparison only. */ - tsd_state_minimal_initialized = 2, - tsd_state_purgatory = 3, - tsd_state_reincarnated = 4, - tsd_state_uninitialized = 5 + /* Common case --> jnz. */ + tsd_state_nominal = 0, + /* Initialized but on slow path. */ + tsd_state_nominal_slow = 1, + /* + * Some thread has changed global state in such a way that all nominal + * threads need to recompute their fast / slow status the next time they + * get a chance. + * + * Any thread can change another thread's status *to* recompute, but + * threads are the only ones who can change their status *from* + * recompute. + */ + tsd_state_nominal_recompute = 2, + /* + * The above nominal states should be lower values. We use + * tsd_nominal_max to separate nominal states from threads in the + * process of being born / dying. + */ + tsd_state_nominal_max = 2, + + /* + * A thread might free() during its death as its only allocator action; + * in such scenarios, we need tsd, but set up in such a way that no + * cleanup is necessary. + */ + tsd_state_minimal_initialized = 3, + /* States during which we know we're in thread death. */ + tsd_state_purgatory = 4, + tsd_state_reincarnated = 5, + /* + * What it says on the tin; tsd that hasn't been initialized. Note + * that even when the tsd struct lives in TLS, when need to keep track + * of stuff like whether or not our pthread destructors have been + * scheduled, so this really truly is different than the nominal state. + */ + tsd_state_uninitialized = 6 }; -/* Manually limit tsd_state_t to a single byte. */ -typedef uint8_t tsd_state_t; +/* + * Some TSD accesses can only be done in a nominal state. To enforce this, we + * wrap TSD member access in a function that asserts on TSD state, and mangle + * field names to prevent touching them accidentally. + */ +#define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n + +#ifdef JEMALLOC_U8_ATOMICS +# define tsd_state_t atomic_u8_t +# define tsd_atomic_load atomic_load_u8 +# define tsd_atomic_store atomic_store_u8 +# define tsd_atomic_exchange atomic_exchange_u8 +#else +# define tsd_state_t atomic_u32_t +# define tsd_atomic_load atomic_load_u32 +# define tsd_atomic_store atomic_store_u32 +# define tsd_atomic_exchange atomic_exchange_u32 +#endif /* The actual tsd. */ struct tsd_s { @@ -117,13 +188,29 @@ struct tsd_s { * module. Access any thread-local state through the getters and * setters below. */ - tsd_state_t state; + + /* + * We manually limit the state to just a single byte. Unless the 8-bit + * atomics are unavailable (which is rare). + */ + tsd_state_t state; #define O(n, t, nt) \ - t use_a_getter_or_setter_instead_##n; + t TSD_MANGLE(n); MALLOC_TSD #undef O }; +JEMALLOC_ALWAYS_INLINE uint8_t +tsd_state_get(tsd_t *tsd) { + /* + * This should be atomic. Unfortunately, compilers right now can't tell + * that this can be done as a memory comparison, and forces a load into + * a register that hurts fast-path performance. + */ + /* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */ + return *(uint8_t *)&tsd->state; +} + /* * Wrapper around tsd_t that makes it possible to avoid implicit conversion * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be @@ -150,15 +237,6 @@ tsdn_tsd(tsdn_t *tsdn) { return &tsdn->tsd; } -void *malloc_tsd_malloc(size_t size); -void malloc_tsd_dalloc(void *wrapper); -void malloc_tsd_cleanup_register(bool (*f)(void)); -tsd_t *malloc_tsd_boot0(void); -void malloc_tsd_boot1(void); -void tsd_cleanup(void *arg); -tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); -void tsd_slow_update(tsd_t *tsd); - /* * We put the platform-specific data declarations and inlines into their own * header files to avoid cluttering this file. They define tsd_boot0, @@ -182,7 +260,7 @@ void tsd_slow_update(tsd_t *tsd); #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get_unsafe(tsd_t *tsd) { \ - return &tsd->use_a_getter_or_setter_instead_##n; \ + return &tsd->TSD_MANGLE(n); \ } MALLOC_TSD #undef O @@ -191,10 +269,16 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get(tsd_t *tsd) { \ - assert(tsd->state == tsd_state_nominal || \ - tsd->state == tsd_state_nominal_slow || \ - tsd->state == tsd_state_reincarnated || \ - tsd->state == tsd_state_minimal_initialized); \ + /* \ + * Because the state might change asynchronously if it's \ + * nominal, we need to make sure that we only read it once. \ + */ \ + uint8_t state = tsd_state_get(tsd); \ + assert(state == tsd_state_nominal || \ + state == tsd_state_nominal_slow || \ + state == tsd_state_nominal_recompute || \ + state == tsd_state_reincarnated || \ + state == tsd_state_minimal_initialized); \ return tsd_##n##p_get_unsafe(tsd); \ } MALLOC_TSD @@ -229,8 +313,8 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE void \ tsd_##n##_set(tsd_t *tsd, t val) { \ - assert(tsd->state != tsd_state_reincarnated && \ - tsd->state != tsd_state_minimal_initialized); \ + assert(tsd_state_get(tsd) != tsd_state_reincarnated && \ + tsd_state_get(tsd) != tsd_state_minimal_initialized); \ *tsd_##n##p_get(tsd) = val; \ } MALLOC_TSD @@ -238,13 +322,18 @@ MALLOC_TSD JEMALLOC_ALWAYS_INLINE void tsd_assert_fast(tsd_t *tsd) { + /* + * Note that our fastness assertion does *not* include global slowness + * counters; it's not in general possible to ensure that they won't + * change asynchronously from underneath us. + */ assert(!malloc_slow && tsd_tcache_enabled_get(tsd) && tsd_reentrancy_level_get(tsd) == 0); } JEMALLOC_ALWAYS_INLINE bool tsd_fast(tsd_t *tsd) { - bool fast = (tsd->state == tsd_state_nominal); + bool fast = (tsd_state_get(tsd) == tsd_state_nominal); if (fast) { tsd_assert_fast(tsd); } @@ -261,7 +350,7 @@ tsd_fetch_impl(bool init, bool minimal) { } assert(tsd != NULL); - if (unlikely(tsd->state != tsd_state_nominal)) { + if (unlikely(tsd_state_get(tsd) != tsd_state_nominal)) { return tsd_fetch_slow(tsd, minimal); } assert(tsd_fast(tsd)); @@ -281,7 +370,7 @@ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_internal_fetch(void) { tsd_t *tsd = tsd_fetch_min(); /* Use reincarnated state to prevent full initialization. */ - tsd->state = tsd_state_reincarnated; + tsd_state_set(tsd, tsd_state_reincarnated); return tsd; } @@ -293,7 +382,7 @@ tsd_fetch(void) { static inline bool tsd_nominal(tsd_t *tsd) { - return (tsd->state <= tsd_state_nominal_max); + return (tsd_state_get(tsd) <= tsd_state_nominal_max); } JEMALLOC_ALWAYS_INLINE tsdn_t * diff --git a/deps/jemalloc/include/jemalloc/internal/tsd_generic.h b/deps/jemalloc/include/jemalloc/internal/tsd_generic.h index 1e52ef767..cf73c0c71 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd_generic.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd_generic.h @@ -77,7 +77,10 @@ tsd_wrapper_get(bool init) { abort(); } else { wrapper->initialized = false; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; } tsd_wrapper_set(wrapper); @@ -107,7 +110,10 @@ tsd_boot1(void) { tsd_boot_wrapper.initialized = false; tsd_cleanup(&tsd_boot_wrapper.val); wrapper->initialized = false; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; tsd_wrapper_set(wrapper); } diff --git a/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index beb467a67..65852d5c1 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -3,8 +3,10 @@ #endif #define JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H -extern __thread tsd_t tsd_tls; -extern __thread bool tsd_initialized; +#define JEMALLOC_TSD_TYPE_ATTR(type) __thread type JEMALLOC_TLS_MODEL + +extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; +extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized; extern bool tsd_booted; /* Initialization/cleanup. */ @@ -47,7 +49,6 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - assert(tsd_booted); return &tsd_tls; } JEMALLOC_ALWAYS_INLINE void diff --git a/deps/jemalloc/include/jemalloc/internal/tsd_tls.h b/deps/jemalloc/include/jemalloc/internal/tsd_tls.h index 0de64b7b8..7d6c805be 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd_tls.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd_tls.h @@ -3,7 +3,9 @@ #endif #define JEMALLOC_INTERNAL_TSD_TLS_H -extern __thread tsd_t tsd_tls; +#define JEMALLOC_TSD_TYPE_ATTR(type) __thread type JEMALLOC_TLS_MODEL + +extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern pthread_key_t tsd_tsd; extern bool tsd_booted; @@ -39,8 +41,7 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * -tsd_get(UNUSED bool init) { - assert(tsd_booted); +tsd_get(bool init) { return &tsd_tls; } diff --git a/deps/jemalloc/include/jemalloc/internal/witness.h b/deps/jemalloc/include/jemalloc/internal/witness.h index 7ace8ae4a..fff9e98cb 100644 --- a/deps/jemalloc/include/jemalloc/internal/witness.h +++ b/deps/jemalloc/include/jemalloc/internal/witness.h @@ -27,9 +27,9 @@ #define WITNESS_RANK_PROF_BT2GCTX 6U #define WITNESS_RANK_PROF_TDATAS 7U #define WITNESS_RANK_PROF_TDATA 8U -#define WITNESS_RANK_PROF_GCTX 9U - -#define WITNESS_RANK_BACKGROUND_THREAD 10U +#define WITNESS_RANK_PROF_LOG 9U +#define WITNESS_RANK_PROF_GCTX 10U +#define WITNESS_RANK_BACKGROUND_THREAD 11U /* * Used as an argument to witness_assert_depth_to_rank() in order to validate @@ -37,18 +37,19 @@ * witness_assert_depth_to_rank() is inclusive rather than exclusive, this * definition can have the same value as the minimally ranked core lock. */ -#define WITNESS_RANK_CORE 11U +#define WITNESS_RANK_CORE 12U -#define WITNESS_RANK_DECAY 11U -#define WITNESS_RANK_TCACHE_QL 12U -#define WITNESS_RANK_EXTENT_GROW 13U -#define WITNESS_RANK_EXTENTS 14U -#define WITNESS_RANK_EXTENT_AVAIL 15U +#define WITNESS_RANK_DECAY 12U +#define WITNESS_RANK_TCACHE_QL 13U +#define WITNESS_RANK_EXTENT_GROW 14U +#define WITNESS_RANK_EXTENTS 15U +#define WITNESS_RANK_EXTENT_AVAIL 16U -#define WITNESS_RANK_EXTENT_POOL 16U -#define WITNESS_RANK_RTREE 17U -#define WITNESS_RANK_BASE 18U -#define WITNESS_RANK_ARENA_LARGE 19U +#define WITNESS_RANK_EXTENT_POOL 17U +#define WITNESS_RANK_RTREE 18U +#define WITNESS_RANK_BASE 19U +#define WITNESS_RANK_ARENA_LARGE 20U +#define WITNESS_RANK_HOOK 21U #define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_BIN WITNESS_RANK_LEAF diff --git a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in index 6d89435c2..11c39181b 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in @@ -4,6 +4,9 @@ /* Defined if alloc_size attribute is supported. */ #undef JEMALLOC_HAVE_ATTR_ALLOC_SIZE +/* Defined if format_arg(...) attribute is supported. */ +#undef JEMALLOC_HAVE_ATTR_FORMAT_ARG + /* Defined if format(gnu_printf, ...) attribute is supported. */ #undef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF diff --git a/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in b/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in index daf9e571b..24a288632 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in @@ -10,6 +10,7 @@ #define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ #define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ #define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" +#define JEMALLOC_VERSION_GID_IDENT @jemalloc_version_gid@ #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -68,6 +69,7 @@ # define JEMALLOC_EXPORT __declspec(dllimport) # endif # endif +# define JEMALLOC_FORMAT_ARG(i) # define JEMALLOC_FORMAT_PRINTF(s, i) # define JEMALLOC_NOINLINE __declspec(noinline) # ifdef __cplusplus @@ -95,6 +97,11 @@ # ifndef JEMALLOC_EXPORT # define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) # endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG +# define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3)) +# else +# define JEMALLOC_FORMAT_ARG(i) +# endif # ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF # define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) # elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) @@ -123,4 +130,4 @@ /* This version of Jemalloc, modified for Redis, has the je_get_defrag_hint() * function. */ -#define JEMALLOC_FRAG_HINT +#define JEMALLOC_FRAG_HINT \ No newline at end of file diff --git a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index f7b175b0a..228e8be0c 100644 --- a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -47,7 +47,7 @@ - + @@ -59,12 +59,14 @@ + + {8D6BB292-9E1C-413D-9F98-4864BDC1514A} @@ -345,4 +347,4 @@ - \ No newline at end of file + diff --git a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index 11cfcd0be..d839515b0 100644 --- a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -37,7 +37,7 @@ Source Files - + Source Files @@ -70,6 +70,9 @@ Source Files + + Source Files + Source Files @@ -97,5 +100,8 @@ Source Files + + Source Files + - \ No newline at end of file + diff --git a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index ed71de8a5..edcceedec 100644 --- a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -47,7 +47,7 @@ - + @@ -59,12 +59,15 @@ + + + {8D6BB292-9E1C-413D-9F98-4864BDC1514A} @@ -344,4 +347,4 @@ - \ No newline at end of file + diff --git a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters index 11cfcd0be..6df726012 100644 --- a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters +++ b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -37,7 +37,7 @@ Source Files - + Source Files @@ -70,6 +70,9 @@ Source Files + + Source Files + Source Files @@ -97,5 +100,11 @@ Source Files + + Source Files + + + Source Files + - \ No newline at end of file + diff --git a/deps/jemalloc/scripts/gen_run_tests.py b/deps/jemalloc/scripts/gen_run_tests.py index a87ecffba..a414f812a 100755 --- a/deps/jemalloc/scripts/gen_run_tests.py +++ b/deps/jemalloc/scripts/gen_run_tests.py @@ -4,6 +4,7 @@ import sys from itertools import combinations from os import uname from multiprocessing import cpu_count +from subprocess import call # Later, we want to test extended vaddr support. Apparently, the "real" way of # checking this is flaky on OS X. @@ -13,13 +14,25 @@ nparallel = cpu_count() * 2 uname = uname()[0] +if "BSD" in uname: + make_cmd = 'gmake' +else: + make_cmd = 'make' + def powerset(items): result = [] for i in xrange(len(items) + 1): result += combinations(items, i) return result -possible_compilers = [('gcc', 'g++'), ('clang', 'clang++')] +possible_compilers = [] +for cc, cxx in (['gcc', 'g++'], ['clang', 'clang++']): + try: + cmd_ret = call([cc, "-v"]) + if cmd_ret == 0: + possible_compilers.append((cc, cxx)) + except: + pass possible_compiler_opts = [ '-m32', ] @@ -27,6 +40,7 @@ possible_config_opts = [ '--enable-debug', '--enable-prof', '--disable-stats', + '--enable-opt-safety-checks', ] if bits_64: possible_config_opts.append('--with-lg-vaddr=56') @@ -39,7 +53,7 @@ possible_malloc_conf_opts = [ ] print 'set -e' -print 'if [ -f Makefile ] ; then make relclean ; fi' +print 'if [ -f Makefile ] ; then %(make_cmd)s relclean ; fi' % {'make_cmd': make_cmd} print 'autoconf' print 'rm -rf run_tests.out' print 'mkdir run_tests.out' @@ -102,11 +116,11 @@ cd run_test_%(ind)d.out echo "==> %(config_line)s" >> run_test.log %(config_line)s >> run_test.log 2>&1 || abort -run_cmd make all tests -run_cmd make check -run_cmd make distclean +run_cmd %(make_cmd)s all tests +run_cmd %(make_cmd)s check +run_cmd %(make_cmd)s distclean EOF -chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line} +chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line, 'make_cmd': make_cmd} ind += 1 print 'for i in `seq 0 %(last_ind)d` ; do echo run_test_${i}.sh ; done | xargs -P %(nparallel)d -n 1 sh' % {'last_ind': ind-1, 'nparallel': nparallel} diff --git a/deps/jemalloc/scripts/gen_travis.py b/deps/jemalloc/scripts/gen_travis.py index 6dd39290c..f1478c62c 100755 --- a/deps/jemalloc/scripts/gen_travis.py +++ b/deps/jemalloc/scripts/gen_travis.py @@ -4,6 +4,7 @@ from itertools import combinations travis_template = """\ language: generic +dist: precise matrix: include: @@ -11,6 +12,7 @@ matrix: before_script: - autoconf + - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script - ./configure ${COMPILER_FLAGS:+ \ CC="$CC $COMPILER_FLAGS" \ CXX="$CXX $COMPILER_FLAGS" } \ @@ -43,6 +45,8 @@ configure_flag_unusuals = [ '--enable-debug', '--enable-prof', '--disable-stats', + '--disable-libdl', + '--enable-opt-safety-checks', ] malloc_conf_unusuals = [ @@ -61,47 +65,85 @@ unusual_combinations_to_test = [] for i in xrange(MAX_UNUSUAL_OPTIONS + 1): unusual_combinations_to_test += combinations(all_unusuals, i) -include_rows = "" -for unusual_combination in unusual_combinations_to_test: - os = os_default - if os_unusual in unusual_combination: - os = os_unusual +gcc_multilib_set = False +# Formats a job from a combination of flags +def format_job(combination): + global gcc_multilib_set - compilers = compilers_default - if compilers_unusual in unusual_combination: - compilers = compilers_unusual + os = os_unusual if os_unusual in combination else os_default + compilers = compilers_unusual if compilers_unusual in combination else compilers_default - compiler_flags = [ - x for x in unusual_combination if x in compiler_flag_unusuals] + compiler_flags = [x for x in combination if x in compiler_flag_unusuals] + configure_flags = [x for x in combination if x in configure_flag_unusuals] + malloc_conf = [x for x in combination if x in malloc_conf_unusuals] - configure_flags = [ - x for x in unusual_combination if x in configure_flag_unusuals] - - malloc_conf = [ - x for x in unusual_combination if x in malloc_conf_unusuals] # Filter out unsupported configurations on OS X. if os == 'osx' and ('dss:primary' in malloc_conf or \ 'percpu_arena:percpu' in malloc_conf or 'background_thread:true' \ in malloc_conf): - continue + return "" if len(malloc_conf) > 0: configure_flags.append('--with-malloc-conf=' + ",".join(malloc_conf)) # Filter out an unsupported configuration - heap profiling on OS X. if os == 'osx' and '--enable-prof' in configure_flags: - continue + return "" # We get some spurious errors when -Warray-bounds is enabled. env_string = ('{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}" ' 'EXTRA_CFLAGS="-Werror -Wno-array-bounds"').format( compilers, " ".join(compiler_flags), " ".join(configure_flags)) - include_rows += ' - os: %s\n' % os - include_rows += ' env: %s\n' % env_string - if '-m32' in unusual_combination and os == 'linux': - include_rows += ' addons:\n' - include_rows += ' apt:\n' - include_rows += ' packages:\n' - include_rows += ' - gcc-multilib\n' + job = "" + job += ' - os: %s\n' % os + job += ' env: %s\n' % env_string + if '-m32' in combination and os == 'linux': + job += ' addons:' + if gcc_multilib_set: + job += ' *gcc_multilib\n' + else: + job += ' &gcc_multilib\n' + job += ' apt:\n' + job += ' packages:\n' + job += ' - gcc-multilib\n' + gcc_multilib_set = True + return job + +include_rows = "" +for combination in unusual_combinations_to_test: + include_rows += format_job(combination) + +# Development build +include_rows += '''\ + # Development build + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" +''' + +# Enable-expermental-smallocx +include_rows += '''\ + # --enable-expermental-smallocx: + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" +''' + +# Valgrind build bots +include_rows += ''' + # Valgrind + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" + addons: + apt: + packages: + - valgrind +''' + +# To enable valgrind on macosx add: +# +# - os: osx +# env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" +# install: brew install valgrind +# +# It currently fails due to: https://github.com/jemalloc/jemalloc/issues/1274 print travis_template % include_rows diff --git a/deps/jemalloc/src/arena.c b/deps/jemalloc/src/arena.c index 5d55bf1a0..ba50e4103 100644 --- a/deps/jemalloc/src/arena.c +++ b/deps/jemalloc/src/arena.c @@ -8,9 +8,11 @@ #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/util.h" +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + /******************************************************************************/ /* Data. */ @@ -40,7 +42,11 @@ const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = { #undef STEP }; -static div_info_t arena_binind_div_info[NBINS]; +static div_info_t arena_binind_div_info[SC_NBINS]; + +size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; +size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; +static unsigned huge_arena_ind; /******************************************************************************/ /* @@ -61,7 +67,7 @@ static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, /******************************************************************************/ void -arena_basic_stats_merge(UNUSED tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, +arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy) { *nthreads += arena_nthreads_get(arena, false); @@ -77,7 +83,8 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_t *bstats, arena_stats_large_t *lstats) { + bin_stats_t *bstats, arena_stats_large_t *lstats, + arena_stats_extents_t *estats) { cassert(config_stats); arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, @@ -94,6 +101,10 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_zu(&astats->retained, extents_npages_get(&arena->extents_retained) << LG_PAGE); + atomic_store_zu(&astats->extent_avail, + atomic_load_zu(&arena->extent_avail_cnt, ATOMIC_RELAXED), + ATOMIC_RELAXED); + arena_stats_accum_u64(&astats->decay_dirty.npurge, arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.decay_dirty.npurge)); @@ -121,8 +132,10 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) + extents_npages_get(&arena->extents_dirty) + extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); + arena_stats_accum_zu(&astats->abandoned_vm, atomic_load_zu( + &arena->stats.abandoned_vm, ATOMIC_RELAXED)); - for (szind_t i = 0; i < NSIZES - NBINS; i++) { + for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) { uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.lstats[i].nmalloc); arena_stats_accum_u64(&lstats[i].nmalloc, nmalloc); @@ -140,12 +153,43 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_u64(&astats->nrequests_large, nmalloc + nrequests); + /* nfill == nmalloc for large currently. */ + arena_stats_accum_u64(&lstats[i].nfills, nmalloc); + arena_stats_accum_u64(&astats->nfills_large, nmalloc); + + uint64_t nflush = arena_stats_read_u64(tsdn, &arena->stats, + &arena->stats.lstats[i].nflushes); + arena_stats_accum_u64(&lstats[i].nflushes, nflush); + arena_stats_accum_u64(&astats->nflushes_large, nflush); + assert(nmalloc >= ndalloc); assert(nmalloc - ndalloc <= SIZE_T_MAX); size_t curlextents = (size_t)(nmalloc - ndalloc); lstats[i].curlextents += curlextents; arena_stats_accum_zu(&astats->allocated_large, - curlextents * sz_index2size(NBINS + i)); + curlextents * sz_index2size(SC_NBINS + i)); + } + + for (pszind_t i = 0; i < SC_NPSIZES; i++) { + size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes, + retained_bytes; + dirty = extents_nextents_get(&arena->extents_dirty, i); + muzzy = extents_nextents_get(&arena->extents_muzzy, i); + retained = extents_nextents_get(&arena->extents_retained, i); + dirty_bytes = extents_nbytes_get(&arena->extents_dirty, i); + muzzy_bytes = extents_nbytes_get(&arena->extents_muzzy, i); + retained_bytes = + extents_nbytes_get(&arena->extents_retained, i); + + atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].nretained, retained, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].dirty_bytes, dirty_bytes, + ATOMIC_RELAXED); + atomic_store_zu(&estats[i].muzzy_bytes, muzzy_bytes, + ATOMIC_RELAXED); + atomic_store_zu(&estats[i].retained_bytes, retained_bytes, + ATOMIC_RELAXED); } arena_stats_unlock(tsdn, &arena->stats); @@ -156,7 +200,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, cache_bin_array_descriptor_t *descriptor; ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { szind_t i = 0; - for (; i < NBINS; i++) { + for (; i < SC_NBINS; i++) { cache_bin_t *tbin = &descriptor->bins_small[i]; arena_stats_accum_zu(&astats->tcache_bytes, tbin->ncached * sz_index2size(i)); @@ -200,8 +244,11 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, nstime_update(&astats->uptime); nstime_subtract(&astats->uptime, &arena->create_time); - for (szind_t i = 0; i < NBINS; i++) { - bin_stats_merge(tsdn, &bstats[i], &arena->bins[i]); + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_stats_merge(tsdn, &bstats[i], + &arena->bins[i].bin_shards[j]); + } } } @@ -236,6 +283,54 @@ arena_slab_reg_alloc(extent_t *slab, const bin_info_t *bin_info) { return ret; } +static void +arena_slab_reg_alloc_batch(extent_t *slab, const bin_info_t *bin_info, + unsigned cnt, void** ptrs) { + arena_slab_data_t *slab_data = extent_slab_data_get(slab); + + assert(extent_nfree_get(slab) >= cnt); + assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); + +#if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) + for (unsigned i = 0; i < cnt; i++) { + size_t regind = bitmap_sfu(slab_data->bitmap, + &bin_info->bitmap_info); + *(ptrs + i) = (void *)((uintptr_t)extent_addr_get(slab) + + (uintptr_t)(bin_info->reg_size * regind)); + } +#else + unsigned group = 0; + bitmap_t g = slab_data->bitmap[group]; + unsigned i = 0; + while (i < cnt) { + while (g == 0) { + g = slab_data->bitmap[++group]; + } + size_t shift = group << LG_BITMAP_GROUP_NBITS; + size_t pop = popcount_lu(g); + if (pop > (cnt - i)) { + pop = cnt - i; + } + + /* + * Load from memory locations only once, outside the + * hot loop below. + */ + uintptr_t base = (uintptr_t)extent_addr_get(slab); + uintptr_t regsize = (uintptr_t)bin_info->reg_size; + while (pop--) { + size_t bit = cfs_lu(&g); + size_t regind = shift + bit; + *(ptrs + i) = (void *)(base + regsize * regind); + + i++; + } + slab_data->bitmap[group] = g; + } +#endif + extent_nfree_sub(slab, cnt); +} + #ifndef JEMALLOC_JET static #endif @@ -291,11 +386,11 @@ arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (usize < SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= NBINS) ? index - NBINS : 0; + hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].nmalloc, 1); @@ -307,11 +402,11 @@ arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (usize < SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= NBINS) ? index - NBINS : 0; + hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].ndalloc, 1); @@ -324,6 +419,11 @@ arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize, arena_large_malloc_stats_update(tsdn, arena, usize); } +static bool +arena_may_have_muzzy(arena_t *arena) { + return (pages_can_purge_lazy && (arena_muzzy_decay_ms_get(arena) != 0)); +} + extent_t * arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero) { @@ -338,7 +438,7 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, extent_t *extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); - if (extent == NULL) { + if (extent == NULL && arena_may_have_muzzy(arena)) { extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); @@ -743,7 +843,7 @@ static size_t arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents, bool all, extent_list_t *decay_extents, bool is_background_thread) { - UNUSED size_t nmadvise, nunmapped; + size_t nmadvise, nunmapped; size_t npurged; if (config_stats) { @@ -834,7 +934,7 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents, npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) { - UNUSED size_t npurged = arena_decay_stashed(tsdn, arena, + size_t npurged = arena_decay_stashed(tsdn, arena, &extent_hooks, decay, extents, all, &decay_extents, is_background_thread); assert(npurged == npurge); @@ -863,7 +963,7 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents, is_background_thread); - UNUSED size_t npages_new; + size_t npages_new; if (epoch_advanced) { /* Backlog is updated on epoch advance. */ npages_new = decay->backlog[SMOOTHSTEP_NSTEPS-1]; @@ -913,11 +1013,17 @@ static void arena_bin_slabs_nonfull_insert(bin_t *bin, extent_t *slab) { assert(extent_nfree_get(slab) > 0); extent_heap_insert(&bin->slabs_nonfull, slab); + if (config_stats) { + bin->stats.nonfull_slabs++; + } } static void arena_bin_slabs_nonfull_remove(bin_t *bin, extent_t *slab) { extent_heap_remove(&bin->slabs_nonfull, slab); + if (config_stats) { + bin->stats.nonfull_slabs--; + } } static extent_t * @@ -928,6 +1034,7 @@ arena_bin_slabs_nonfull_tryget(bin_t *bin) { } if (config_stats) { bin->stats.reslabs++; + bin->stats.nonfull_slabs--; } return slab; } @@ -954,6 +1061,37 @@ arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, extent_t *slab) { extent_list_remove(&bin->slabs_full, slab); } +static void +arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { + extent_t *slab; + + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + if (bin->slabcur != NULL) { + slab = bin->slabcur; + bin->slabcur = NULL; + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != NULL) { + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + for (slab = extent_list_first(&bin->slabs_full); slab != NULL; + slab = extent_list_first(&bin->slabs_full)) { + arena_bin_slabs_full_remove(arena, bin, slab); + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + if (config_stats) { + bin->stats.curregs = 0; + bin->stats.curslabs = 0; + } + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); +} + void arena_reset(tsd_t *tsd, arena_t *arena) { /* @@ -983,7 +1121,7 @@ arena_reset(tsd_t *tsd, arena_t *arena) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); if (config_stats || (config_prof && opt_prof)) { usize = sz_index2size(alloc_ctx.szind); @@ -999,35 +1137,11 @@ arena_reset(tsd_t *tsd, arena_t *arena) { malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); /* Bins. */ - for (unsigned i = 0; i < NBINS; i++) { - extent_t *slab; - bin_t *bin = &arena->bins[i]; - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (bin->slabcur != NULL) { - slab = bin->slabcur; - bin->slabcur = NULL; - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + arena_bin_reset(tsd, arena, + &arena->bins[i].bin_shards[j]); } - while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != - NULL) { - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - for (slab = extent_list_first(&bin->slabs_full); slab != NULL; - slab = extent_list_first(&bin->slabs_full)) { - arena_bin_slabs_full_remove(arena, bin, slab); - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - if (config_stats) { - bin->stats.curregs = 0; - bin->stats.curslabs = 0; - } - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); } atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); @@ -1112,7 +1226,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, } static extent_t * -arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, +arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard, const bin_info_t *bin_info) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -1124,7 +1238,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); - if (slab == NULL) { + if (slab == NULL && arena_may_have_muzzy(arena)) { slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); @@ -1140,7 +1254,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, /* Initialize slab internals. */ arena_slab_data_t *slab_data = extent_slab_data_get(slab); - extent_nfree_set(slab, bin_info->nregs); + extent_nfree_binshard_set(slab, bin_info->nregs, binshard); bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false); arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); @@ -1150,7 +1264,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, static extent_t * arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind) { + szind_t binind, unsigned binshard) { extent_t *slab; const bin_info_t *bin_info; @@ -1166,7 +1280,7 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, /* Allocate a new slab. */ malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ - slab = arena_slab_alloc(tsdn, arena, binind, bin_info); + slab = arena_slab_alloc(tsdn, arena, binind, binshard, bin_info); /********************************/ malloc_mutex_lock(tsdn, &bin->lock); if (slab != NULL) { @@ -1193,7 +1307,7 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, /* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */ static void * arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind) { + szind_t binind, unsigned binshard) { const bin_info_t *bin_info; extent_t *slab; @@ -1202,7 +1316,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, arena_bin_slabs_full_insert(arena, bin, bin->slabcur); bin->slabcur = NULL; } - slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind); + slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind, binshard); if (bin->slabcur != NULL) { /* * Another thread updated slabcur while this one ran without the @@ -1246,46 +1360,75 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, return arena_slab_reg_alloc(slab, bin_info); } +/* Choose a bin shard and return the locked bin. */ +bin_t * +arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard) { + bin_t *bin; + if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { + *binshard = 0; + } else { + *binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind]; + } + assert(*binshard < bin_infos[binind].n_shards); + bin = &arena->bins[binind].bin_shards[*binshard]; + malloc_mutex_lock(tsdn, &bin->lock); + + return bin; +} + void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) { - unsigned i, nfill; - bin_t *bin; + unsigned i, nfill, cnt; assert(tbin->ncached == 0); if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) { prof_idump(tsdn); } - bin = &arena->bins[binind]; - malloc_mutex_lock(tsdn, &bin->lock); + + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); + for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> - tcache->lg_fill_div[binind]); i < nfill; i++) { + tcache->lg_fill_div[binind]); i < nfill; i += cnt) { extent_t *slab; - void *ptr; if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { - ptr = arena_slab_reg_alloc(slab, &bin_infos[binind]); + unsigned tofill = nfill - i; + cnt = tofill < extent_nfree_get(slab) ? + tofill : extent_nfree_get(slab); + arena_slab_reg_alloc_batch( + slab, &bin_infos[binind], cnt, + tbin->avail - nfill + i); } else { - ptr = arena_bin_malloc_hard(tsdn, arena, bin, binind); - } - if (ptr == NULL) { + cnt = 1; + void *ptr = arena_bin_malloc_hard(tsdn, arena, bin, + binind, binshard); /* * OOM. tbin->avail isn't yet filled down to its first * element, so the successful allocations (if any) must * be moved just before tbin->avail before bailing out. */ - if (i > 0) { - memmove(tbin->avail - i, tbin->avail - nfill, - i * sizeof(void *)); + if (ptr == NULL) { + if (i > 0) { + memmove(tbin->avail - i, + tbin->avail - nfill, + i * sizeof(void *)); + } + break; } - break; + /* Insert such that low regions get used first. */ + *(tbin->avail - nfill + i) = ptr; } if (config_fill && unlikely(opt_junk_alloc)) { - arena_alloc_junk_small(ptr, &bin_infos[binind], true); + for (unsigned j = 0; j < cnt; j++) { + void* ptr = *(tbin->avail - nfill + i + j); + arena_alloc_junk_small(ptr, &bin_infos[binind], + true); + } } - /* Insert such that low regions get used first. */ - *(tbin->avail - nfill + i) = ptr; } if (config_stats) { bin->stats.nmalloc += i; @@ -1320,15 +1463,15 @@ arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { size_t usize; extent_t *slab; - assert(binind < NBINS); - bin = &arena->bins[binind]; + assert(binind < SC_NBINS); usize = sz_index2size(binind); + unsigned binshard; + bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); - malloc_mutex_lock(tsdn, &bin->lock); if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { ret = arena_slab_reg_alloc(slab, &bin_infos[binind]); } else { - ret = arena_bin_malloc_hard(tsdn, arena, bin, binind); + ret = arena_bin_malloc_hard(tsdn, arena, bin, binind, binshard); } if (ret == NULL) { @@ -1373,13 +1516,13 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, assert(!tsdn_null(tsdn) || arena != NULL); if (likely(!tsdn_null(tsdn))) { - arena = arena_choose(tsdn_tsd(tsdn), arena); + arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, size); } if (unlikely(arena == NULL)) { return NULL; } - if (likely(size <= SMALL_MAXCLASS)) { + if (likely(size <= SC_SMALL_MAXCLASS)) { return arena_malloc_small(tsdn, arena, ind, zero); } return large_malloc(tsdn, arena, sz_index2size(ind), zero); @@ -1390,8 +1533,9 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero, tcache_t *tcache) { void *ret; - if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE - && (usize & PAGE_MASK) == 0))) { + if (usize <= SC_SMALL_MAXCLASS + && (alignment < PAGE + || (alignment == PAGE && (usize & PAGE_MASK) == 0))) { /* Small; alignment doesn't require special slab placement. */ ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), zero, tcache, true); @@ -1406,11 +1550,15 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, } void -arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize) { +arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) { cassert(config_prof); assert(ptr != NULL); - assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); - assert(usize <= SMALL_MAXCLASS); + assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); + assert(usize <= SC_SMALL_MAXCLASS); + + if (config_opt_safety_checks) { + safety_check_set_redzone(ptr, usize, SC_LARGE_MINCLASS); + } rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -1434,15 +1582,15 @@ arena_prof_demote(tsdn_t *tsdn, extent_t *extent, const void *ptr) { cassert(config_prof); assert(ptr != NULL); - extent_szind_set(extent, NBINS); + extent_szind_set(extent, SC_NBINS); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, - NBINS, false); + SC_NBINS, false); - assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); + assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); - return LARGE_MINCLASS; + return SC_LARGE_MINCLASS; } void @@ -1452,10 +1600,19 @@ arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, assert(opt_prof); extent_t *extent = iealloc(tsdn, ptr); - size_t usize = arena_prof_demote(tsdn, extent, ptr); - if (usize <= tcache_maxclass) { + size_t usize = extent_usize_get(extent); + size_t bumped_usize = arena_prof_demote(tsdn, extent, ptr); + if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) { + /* + * Currently, we only do redzoning for small sampled + * allocations. + */ + assert(bumped_usize == SC_LARGE_MINCLASS); + safety_check_verify_redzone(ptr, usize, bumped_usize); + } + if (bumped_usize <= tcache_maxclass && tcache != NULL) { tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, - sz_size2index(usize), slow_path); + sz_size2index(bumped_usize), slow_path); } else { large_dalloc(tsdn, extent); } @@ -1499,7 +1656,7 @@ arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } static void -arena_bin_lower_slab(UNUSED tsdn_t *tsdn, arena_t *arena, extent_t *slab, +arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, bin_t *bin) { assert(extent_nfree_get(slab) > 0); @@ -1526,11 +1683,9 @@ arena_bin_lower_slab(UNUSED tsdn_t *tsdn, arena_t *arena, extent_t *slab, } static void -arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, - void *ptr, bool junked) { +arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *slab, void *ptr, bool junked) { arena_slab_data_t *slab_data = extent_slab_data_get(slab); - szind_t binind = extent_szind_get(slab); - bin_t *bin = &arena->bins[binind]; const bin_info_t *bin_info = &bin_infos[binind]; if (!junked && config_fill && unlikely(opt_junk_free)) { @@ -1554,18 +1709,21 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } void -arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - void *ptr) { - arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, true); +arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *extent, void *ptr) { + arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, + true); } static void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { szind_t binind = extent_szind_get(extent); - bin_t *bin = &arena->bins[binind]; + unsigned binshard = extent_binshard_get(extent); + bin_t *bin = &arena->bins[binind].bin_shards[binshard]; malloc_mutex_lock(tsdn, &bin->lock); - arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, false); + arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, + false); malloc_mutex_unlock(tsdn, &bin->lock); } @@ -1580,38 +1738,48 @@ arena_dalloc_small(tsdn_t *tsdn, void *ptr) { bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero) { + size_t extra, bool zero, size_t *newsize) { + bool ret; /* Calls with non-zero extra had to clamp extra. */ - assert(extra == 0 || size + extra <= LARGE_MAXCLASS); - - if (unlikely(size > LARGE_MAXCLASS)) { - return true; - } + assert(extra == 0 || size + extra <= SC_LARGE_MAXCLASS); extent_t *extent = iealloc(tsdn, ptr); + if (unlikely(size > SC_LARGE_MAXCLASS)) { + ret = true; + goto done; + } + size_t usize_min = sz_s2u(size); size_t usize_max = sz_s2u(size + extra); - if (likely(oldsize <= SMALL_MAXCLASS && usize_min <= SMALL_MAXCLASS)) { + if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min + <= SC_SMALL_MAXCLASS)) { /* * Avoid moving the allocation if the size class can be left the * same. */ assert(bin_infos[sz_size2index(oldsize)].reg_size == oldsize); - if ((usize_max > SMALL_MAXCLASS || sz_size2index(usize_max) != - sz_size2index(oldsize)) && (size > oldsize || usize_max < - oldsize)) { - return true; + if ((usize_max > SC_SMALL_MAXCLASS + || sz_size2index(usize_max) != sz_size2index(oldsize)) + && (size > oldsize || usize_max < oldsize)) { + ret = true; + goto done; } arena_decay_tick(tsdn, extent_arena_get(extent)); - return false; - } else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) { - return large_ralloc_no_move(tsdn, extent, usize_min, usize_max, + ret = false; + } else if (oldsize >= SC_LARGE_MINCLASS + && usize_max >= SC_LARGE_MINCLASS) { + ret = large_ralloc_no_move(tsdn, extent, usize_min, usize_max, zero); + } else { + ret = true; } +done: + assert(extent == iealloc(tsdn, ptr)); + *newsize = extent_usize_get(extent); - return true; + return ret; } static void * @@ -1622,7 +1790,7 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, zero, tcache, true); } usize = sz_sa2u(usize, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } return ipalloct(tsdn, usize, alignment, zero, tcache, arena); @@ -1630,22 +1798,30 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, void * arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache) { + size_t size, size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args) { size_t usize = sz_s2u(size); - if (unlikely(usize == 0 || size > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) { return NULL; } - if (likely(usize <= SMALL_MAXCLASS)) { + if (likely(usize <= SC_SMALL_MAXCLASS)) { /* Try to avoid moving the allocation. */ - if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero)) { + UNUSED size_t newsize; + if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero, + &newsize)) { + hook_invoke_expand(hook_args->is_realloc + ? hook_expand_realloc : hook_expand_rallocx, + ptr, oldsize, usize, (uintptr_t)ptr, + hook_args->args); return ptr; } } - if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) { - return large_ralloc(tsdn, arena, iealloc(tsdn, ptr), usize, - alignment, zero, tcache); + if (oldsize >= SC_LARGE_MINCLASS + && usize >= SC_LARGE_MINCLASS) { + return large_ralloc(tsdn, arena, ptr, usize, + alignment, zero, tcache, hook_args); } /* @@ -1658,11 +1834,16 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, return NULL; } + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + /* * Junk/zero-filling were already done by * ipalloc()/arena_malloc(). */ - size_t copysize = (usize < oldsize) ? usize : oldsize; memcpy(ret, ptr, copysize); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); @@ -1720,8 +1901,7 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, if (new_limit != NULL) { size_t limit = *new_limit; /* Grow no more than the new limit. */ - if ((new_ind = sz_psz2ind(limit + 1) - 1) > - EXTENT_GROW_MAX_PIND) { + if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) { return true; } } @@ -1773,7 +1953,12 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } } - arena = (arena_t *)base_alloc(tsdn, base, sizeof(arena_t), CACHELINE); + unsigned nbins_total = 0; + for (i = 0; i < SC_NBINS; i++) { + nbins_total += bin_infos[i].n_shards; + } + size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total; + arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); if (arena == NULL) { goto label_error; } @@ -1865,7 +2050,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } arena->extent_grow_next = sz_psz2ind(HUGEPAGE); - arena->retain_grow_limit = EXTENT_GROW_MAX_PIND; + arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS); if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow", WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { goto label_error; @@ -1878,12 +2063,20 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } /* Initialize bins. */ - for (i = 0; i < NBINS; i++) { - bool err = bin_init(&arena->bins[i]); - if (err) { - goto label_error; + uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t); + atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); + for (i = 0; i < SC_NBINS; i++) { + unsigned nshards = bin_infos[i].n_shards; + arena->bins[i].bin_shards = (bin_t *)bin_addr; + bin_addr += nshards * sizeof(bin_t); + for (unsigned j = 0; j < nshards; j++) { + bool err = bin_init(&arena->bins[i].bin_shards[j]); + if (err) { + goto label_error; + } } } + assert(bin_addr == (uintptr_t)arena + arena_size); arena->base = base; /* Set arena before creating background threads. */ @@ -1900,8 +2093,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ assert(!tsdn_null(tsdn)); pre_reentrancy(tsdn_tsd(tsdn), arena); - if (hooks_arena_new_hook) { - hooks_arena_new_hook(); + if (test_hooks_arena_new_hook) { + test_hooks_arena_new_hook(); } post_reentrancy(tsdn_tsd(tsdn)); } @@ -1914,20 +2107,75 @@ label_error: return NULL; } +arena_t * +arena_choose_huge(tsd_t *tsd) { + /* huge_arena_ind can be 0 during init (will use a0). */ + if (huge_arena_ind == 0) { + assert(!malloc_initialized()); + } + + arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false); + if (huge_arena == NULL) { + /* Create the huge arena on demand. */ + assert(huge_arena_ind != 0); + huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true); + if (huge_arena == NULL) { + return NULL; + } + /* + * Purge eagerly for huge allocations, because: 1) number of + * huge allocations is usually small, which means ticker based + * decay is not reliable; and 2) less immediate reuse is + * expected for huge allocations. + */ + if (arena_dirty_decay_ms_default_get() > 0) { + arena_dirty_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); + } + if (arena_muzzy_decay_ms_default_get() > 0) { + arena_muzzy_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); + } + } + + return huge_arena; +} + +bool +arena_init_huge(void) { + bool huge_enabled; + + /* The threshold should be large size class. */ + if (opt_oversize_threshold > SC_LARGE_MAXCLASS || + opt_oversize_threshold < SC_LARGE_MINCLASS) { + opt_oversize_threshold = 0; + oversize_threshold = SC_LARGE_MAXCLASS + PAGE; + huge_enabled = false; + } else { + /* Reserve the index for the huge arena. */ + huge_arena_ind = narenas_total_get(); + oversize_threshold = opt_oversize_threshold; + huge_enabled = true; + } + + return huge_enabled; +} + +bool +arena_is_huge(unsigned arena_ind) { + if (huge_arena_ind == 0) { + return false; + } + return (arena_ind == huge_arena_ind); +} + void -arena_boot(void) { +arena_boot(sc_data_t *sc_data) { arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms); -#define REGIND_bin_yes(index, reg_size) \ - div_init(&arena_binind_div_info[(index)], (reg_size)); -#define REGIND_bin_no(index, reg_size) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ - lg_delta_lookup) \ - REGIND_bin_##bin(index, (1U<sc[i]; + div_init(&arena_binind_div_info[i], + (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta)); + } } void @@ -1972,8 +2220,10 @@ arena_prefork6(tsdn_t *tsdn, arena_t *arena) { void arena_prefork7(tsdn_t *tsdn, arena_t *arena) { - for (unsigned i = 0; i < NBINS; i++) { - bin_prefork(tsdn, &arena->bins[i]); + for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_prefork(tsdn, &arena->bins[i].bin_shards[j]); + } } } @@ -1981,8 +2231,11 @@ void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { unsigned i; - for (i = 0; i < NBINS; i++) { - bin_postfork_parent(tsdn, &arena->bins[i]); + for (i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_postfork_parent(tsdn, + &arena->bins[i].bin_shards[j]); + } } malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); base_postfork_parent(tsdn, arena->base); @@ -2025,8 +2278,10 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { } } - for (i = 0; i < NBINS; i++) { - bin_postfork_child(tsdn, &arena->bins[i]); + for (i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_postfork_child(tsdn, &arena->bins[i].bin_shards[j]); + } } malloc_mutex_postfork_child(tsdn, &arena->large_mtx); base_postfork_child(tsdn, arena->base); diff --git a/deps/jemalloc/src/background_thread.c b/deps/jemalloc/src/background_thread.c index 457669c9e..57b9b256b 100644 --- a/deps/jemalloc/src/background_thread.c +++ b/deps/jemalloc/src/background_thread.c @@ -4,6 +4,8 @@ #include "jemalloc/internal/assert.h" +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + /******************************************************************************/ /* Data. */ @@ -11,7 +13,7 @@ #define BACKGROUND_THREAD_DEFAULT false /* Read-only after initialization. */ bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; -size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT; +size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; /* Used for thread creation, termination and stats. */ malloc_mutex_t background_thread_lock; @@ -22,13 +24,9 @@ size_t max_background_threads; /* Thread info per-index. */ background_thread_info_t *background_thread_info; -/* False if no necessary runtime support. */ -bool can_enable_background_thread; - /******************************************************************************/ #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER -#include static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, void *(*)(void *), void *__restrict); @@ -81,7 +79,7 @@ background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { } static inline bool -set_current_thread_affinity(UNUSED int cpu) { +set_current_thread_affinity(int cpu) { #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) cpu_set_t cpuset; CPU_ZERO(&cpuset); @@ -510,6 +508,8 @@ background_thread_entry(void *ind_arg) { assert(thread_ind < max_background_threads); #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP pthread_setname_np(pthread_self(), "jemalloc_bg_thd"); +#elif defined(__FreeBSD__) + pthread_set_name_np(pthread_self(), "jemalloc_bg_thd"); #endif if (opt_percpu_arena != percpu_arena_disabled) { set_current_thread_affinity((int)thread_ind); @@ -534,9 +534,8 @@ background_thread_init(tsd_t *tsd, background_thread_info_t *info) { n_background_threads++; } -/* Create a new background thread if needed. */ -bool -background_thread_create(tsd_t *tsd, unsigned arena_ind) { +static bool +background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { assert(have_background_thread); malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); @@ -589,6 +588,19 @@ background_thread_create(tsd_t *tsd, unsigned arena_ind) { return false; } +/* Create a new background thread if needed. */ +bool +background_thread_create(tsd_t *tsd, unsigned arena_ind) { + assert(have_background_thread); + + bool ret; + malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); + ret = background_thread_create_locked(tsd, arena_ind); + malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); + + return ret; +} + bool background_threads_enable(tsd_t *tsd) { assert(n_background_threads == 0); @@ -622,7 +634,7 @@ background_threads_enable(tsd_t *tsd) { } } - return background_thread_create(tsd, 0); + return background_thread_create_locked(tsd, 0); } bool @@ -813,21 +825,34 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { #undef BILLION #undef BACKGROUND_THREAD_MIN_INTERVAL_NS +#ifdef JEMALLOC_HAVE_DLSYM +#include +#endif + static bool pthread_create_fptr_init(void) { if (pthread_create_fptr != NULL) { return false; } + /* + * Try the next symbol first, because 1) when use lazy_lock we have a + * wrapper for pthread_create; and 2) application may define its own + * wrapper as well (and can call malloc within the wrapper). + */ +#ifdef JEMALLOC_HAVE_DLSYM pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); +#else + pthread_create_fptr = NULL; +#endif if (pthread_create_fptr == NULL) { - can_enable_background_thread = false; - if (config_lazy_lock || opt_background_thread) { + if (config_lazy_lock) { malloc_write(": Error in dlsym(RTLD_NEXT, " "\"pthread_create\")\n"); abort(); + } else { + /* Fall back to the default symbol. */ + pthread_create_fptr = pthread_create; } - } else { - can_enable_background_thread = true; } return false; @@ -872,9 +897,8 @@ background_thread_boot1(tsdn_t *tsdn) { assert(have_background_thread); assert(narenas_total_get() > 0); - if (opt_max_background_threads == MAX_BACKGROUND_THREAD_LIMIT && - ncpus < MAX_BACKGROUND_THREAD_LIMIT) { - opt_max_background_threads = ncpus; + if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) { + opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD; } max_background_threads = opt_max_background_threads; diff --git a/deps/jemalloc/src/base.c b/deps/jemalloc/src/base.c index b0324b5d7..f3c61661a 100644 --- a/deps/jemalloc/src/base.c +++ b/deps/jemalloc/src/base.c @@ -262,8 +262,8 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks, */ size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size + usize)); - pszind_t pind_next = (*pind_last + 1 < NPSIZES) ? *pind_last + 1 : - *pind_last; + pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ? + *pind_last + 1 : *pind_last; size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next)); size_t block_size = (min_block_size > next_block_size) ? min_block_size : next_block_size; @@ -372,7 +372,7 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { base->extent_sn_next = extent_sn_next; base->blocks = block; base->auto_thp_switched = false; - for (szind_t i = 0; i < NSIZES; i++) { + for (szind_t i = 0; i < SC_NSIZES; i++) { extent_heap_new(&base->avail[i]); } if (config_stats) { @@ -426,7 +426,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment, extent_t *extent = NULL; malloc_mutex_lock(tsdn, &base->mtx); - for (szind_t i = sz_size2index(asize); i < NSIZES; i++) { + for (szind_t i = sz_size2index(asize); i < SC_NSIZES; i++) { extent = extent_heap_remove_first(&base->avail[i]); if (extent != NULL) { /* Use existing space. */ diff --git a/deps/jemalloc/src/bin.c b/deps/jemalloc/src/bin.c index 0886bc4ea..bca6b12c3 100644 --- a/deps/jemalloc/src/bin.c +++ b/deps/jemalloc/src/bin.c @@ -1,23 +1,68 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" +#include "jemalloc/internal/assert.h" #include "jemalloc/internal/bin.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/witness.h" -const bin_info_t bin_infos[NBINS] = { -#define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \ - {reg_size, slab_size, nregs, BITMAP_INFO_INITIALIZER(nregs)}, -#define BIN_INFO_bin_no(reg_size, slab_size, nregs) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ - lg_delta_lookup) \ - BIN_INFO_bin_##bin((1U<sc[i]; + bin_info->reg_size = ((size_t)1U << sc->lg_base) + + ((size_t)sc->ndelta << sc->lg_delta); + bin_info->slab_size = (sc->pgs << LG_PAGE); + bin_info->nregs = + (uint32_t)(bin_info->slab_size / bin_info->reg_size); + bin_info->n_shards = bin_shard_sizes[i]; + bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER( + bin_info->nregs); + bin_info->bitmap_info = bitmap_info; + } +} + +bool +bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards) { + if (nshards > BIN_SHARDS_MAX || nshards == 0) { + return true; + } + + if (start_size > SC_SMALL_MAXCLASS) { + return false; + } + if (end_size > SC_SMALL_MAXCLASS) { + end_size = SC_SMALL_MAXCLASS; + } + + /* Compute the index since this may happen before sz init. */ + szind_t ind1 = sz_size2index_compute(start_size); + szind_t ind2 = sz_size2index_compute(end_size); + for (unsigned i = ind1; i <= ind2; i++) { + bin_shard_sizes[i] = (unsigned)nshards; + } + + return false; +} + +void +bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) { + /* Load the default number of shards. */ + for (unsigned i = 0; i < SC_NBINS; i++) { + bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT; + } +} + +void +bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { + assert(sc_data->initialized); + bin_infos_init(sc_data, bin_shard_sizes, bin_infos); +} bool bin_init(bin_t *bin) { diff --git a/deps/jemalloc/src/ckh.c b/deps/jemalloc/src/ckh.c index e95e0a3ed..1bf6df5a1 100644 --- a/deps/jemalloc/src/ckh.c +++ b/deps/jemalloc/src/ckh.c @@ -275,7 +275,8 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) { lg_curcells++; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } @@ -320,7 +321,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { lg_prevbuckets = ckh->lg_curbuckets; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return; } tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL, @@ -396,7 +397,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh->keycomp = keycomp; usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } diff --git a/deps/jemalloc/src/ctl.c b/deps/jemalloc/src/ctl.c index 1e713a3d1..48afaa61f 100644 --- a/deps/jemalloc/src/ctl.c +++ b/deps/jemalloc/src/ctl.c @@ -8,7 +8,7 @@ #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/util.h" /******************************************************************************/ @@ -72,6 +72,7 @@ CTL_PROTO(config_debug) CTL_PROTO(config_fill) CTL_PROTO(config_lazy_lock) CTL_PROTO(config_malloc_conf) +CTL_PROTO(config_opt_safety_checks) CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) @@ -80,11 +81,13 @@ CTL_PROTO(config_utrace) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) CTL_PROTO(opt_abort_conf) +CTL_PROTO(opt_confirm_conf) CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) CTL_PROTO(opt_narenas) CTL_PROTO(opt_percpu_arena) +CTL_PROTO(opt_oversize_threshold) CTL_PROTO(opt_background_thread) CTL_PROTO(opt_max_background_threads) CTL_PROTO(opt_dirty_decay_ms) @@ -126,6 +129,7 @@ INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_slab_size) +CTL_PROTO(arenas_bin_i_nshards) INDEX_PROTO(arenas_bin_i) CTL_PROTO(arenas_lextent_i_size) INDEX_PROTO(arenas_lextent_i) @@ -147,14 +151,20 @@ CTL_PROTO(prof_gdump) CTL_PROTO(prof_reset) CTL_PROTO(prof_interval) CTL_PROTO(lg_prof_sample) +CTL_PROTO(prof_log_start) +CTL_PROTO(prof_log_stop) CTL_PROTO(stats_arenas_i_small_allocated) CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_ndalloc) CTL_PROTO(stats_arenas_i_small_nrequests) +CTL_PROTO(stats_arenas_i_small_nfills) +CTL_PROTO(stats_arenas_i_small_nflushes) CTL_PROTO(stats_arenas_i_large_allocated) CTL_PROTO(stats_arenas_i_large_nmalloc) CTL_PROTO(stats_arenas_i_large_ndalloc) CTL_PROTO(stats_arenas_i_large_nrequests) +CTL_PROTO(stats_arenas_i_large_nfills) +CTL_PROTO(stats_arenas_i_large_nflushes) CTL_PROTO(stats_arenas_i_bins_j_nmalloc) CTL_PROTO(stats_arenas_i_bins_j_ndalloc) CTL_PROTO(stats_arenas_i_bins_j_nrequests) @@ -164,12 +174,20 @@ CTL_PROTO(stats_arenas_i_bins_j_nflushes) CTL_PROTO(stats_arenas_i_bins_j_nslabs) CTL_PROTO(stats_arenas_i_bins_j_nreslabs) CTL_PROTO(stats_arenas_i_bins_j_curslabs) +CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs) INDEX_PROTO(stats_arenas_i_bins_j) CTL_PROTO(stats_arenas_i_lextents_j_nmalloc) CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) CTL_PROTO(stats_arenas_i_lextents_j_nrequests) CTL_PROTO(stats_arenas_i_lextents_j_curlextents) INDEX_PROTO(stats_arenas_i_lextents_j) +CTL_PROTO(stats_arenas_i_extents_j_ndirty) +CTL_PROTO(stats_arenas_i_extents_j_nmuzzy) +CTL_PROTO(stats_arenas_i_extents_j_nretained) +CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) +CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) +CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) +INDEX_PROTO(stats_arenas_i_extents_j) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_uptime) CTL_PROTO(stats_arenas_i_dss) @@ -180,6 +198,7 @@ CTL_PROTO(stats_arenas_i_pdirty) CTL_PROTO(stats_arenas_i_pmuzzy) CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_retained) +CTL_PROTO(stats_arenas_i_extent_avail) CTL_PROTO(stats_arenas_i_dirty_npurge) CTL_PROTO(stats_arenas_i_dirty_nmadvise) CTL_PROTO(stats_arenas_i_dirty_purged) @@ -191,6 +210,7 @@ CTL_PROTO(stats_arenas_i_internal) CTL_PROTO(stats_arenas_i_metadata_thp) CTL_PROTO(stats_arenas_i_tcache_bytes) CTL_PROTO(stats_arenas_i_resident) +CTL_PROTO(stats_arenas_i_abandoned_vm) INDEX_PROTO(stats_arenas_i) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) @@ -202,6 +222,12 @@ CTL_PROTO(stats_metadata_thp) CTL_PROTO(stats_resident) CTL_PROTO(stats_mapped) CTL_PROTO(stats_retained) +CTL_PROTO(experimental_hooks_install) +CTL_PROTO(experimental_hooks_remove) +CTL_PROTO(experimental_utilization_query) +CTL_PROTO(experimental_utilization_batch_query) +CTL_PROTO(experimental_arenas_i_pactivep) +INDEX_PROTO(experimental_arenas_i) #define MUTEX_STATS_CTL_PROTO_GEN(n) \ CTL_PROTO(stats_##n##_num_ops) \ @@ -270,6 +296,7 @@ static const ctl_named_node_t config_node[] = { {NAME("fill"), CTL(config_fill)}, {NAME("lazy_lock"), CTL(config_lazy_lock)}, {NAME("malloc_conf"), CTL(config_malloc_conf)}, + {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)}, {NAME("prof"), CTL(config_prof)}, {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, @@ -281,11 +308,13 @@ static const ctl_named_node_t config_node[] = { static const ctl_named_node_t opt_node[] = { {NAME("abort"), CTL(opt_abort)}, {NAME("abort_conf"), CTL(opt_abort_conf)}, + {NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("metadata_thp"), CTL(opt_metadata_thp)}, {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("percpu_arena"), CTL(opt_percpu_arena)}, + {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, {NAME("background_thread"), CTL(opt_background_thread)}, {NAME("max_background_threads"), CTL(opt_max_background_threads)}, {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, @@ -341,7 +370,8 @@ static const ctl_indexed_node_t arena_node[] = { static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("slab_size"), CTL(arenas_bin_i_slab_size)} + {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, + {NAME("nshards"), CTL(arenas_bin_i_nshards)} }; static const ctl_named_node_t super_arenas_bin_i_node[] = { {NAME(""), CHILD(named, arenas_bin_i)} @@ -385,21 +415,26 @@ static const ctl_named_node_t prof_node[] = { {NAME("gdump"), CTL(prof_gdump)}, {NAME("reset"), CTL(prof_reset)}, {NAME("interval"), CTL(prof_interval)}, - {NAME("lg_sample"), CTL(lg_prof_sample)} + {NAME("lg_sample"), CTL(lg_prof_sample)}, + {NAME("log_start"), CTL(prof_log_start)}, + {NAME("log_stop"), CTL(prof_log_stop)} }; - static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} + {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}, + {NAME("nfills"), CTL(stats_arenas_i_small_nfills)}, + {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)} }; static const ctl_named_node_t stats_arenas_i_large_node[] = { {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} + {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}, + {NAME("nfills"), CTL(stats_arenas_i_large_nfills)}, + {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)} }; #define MUTEX_PROF_DATA_NODE(prefix) \ @@ -433,6 +468,7 @@ static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, + {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)}, {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} }; @@ -458,6 +494,23 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { {INDEX(stats_arenas_i_lextents_j)} }; +static const ctl_named_node_t stats_arenas_i_extents_j_node[] = { + {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, + {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, + {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, + {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, + {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, + {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)} +}; + +static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_extents_j)} +}; + +static const ctl_indexed_node_t stats_arenas_i_extents_node[] = { + {INDEX(stats_arenas_i_extents_j)} +}; + #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) MUTEX_PROF_ARENA_MUTEXES #undef OP @@ -479,6 +532,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("retained"), CTL(stats_arenas_i_retained)}, + {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, @@ -490,10 +544,12 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, {NAME("resident"), CTL(stats_arenas_i_resident)}, + {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)}, {NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, + {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} }; static const ctl_named_node_t super_stats_arenas_i_node[] = { @@ -536,6 +592,33 @@ static const ctl_named_node_t stats_node[] = { {NAME("arenas"), CHILD(indexed, stats_arenas)} }; +static const ctl_named_node_t experimental_hooks_node[] = { + {NAME("install"), CTL(experimental_hooks_install)}, + {NAME("remove"), CTL(experimental_hooks_remove)} +}; + +static const ctl_named_node_t experimental_utilization_node[] = { + {NAME("query"), CTL(experimental_utilization_query)}, + {NAME("batch_query"), CTL(experimental_utilization_batch_query)} +}; + +static const ctl_named_node_t experimental_arenas_i_node[] = { + {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)} +}; +static const ctl_named_node_t super_experimental_arenas_i_node[] = { + {NAME(""), CHILD(named, experimental_arenas_i)} +}; + +static const ctl_indexed_node_t experimental_arenas_node[] = { + {INDEX(experimental_arenas_i)} +}; + +static const ctl_named_node_t experimental_node[] = { + {NAME("hooks"), CHILD(named, experimental_hooks)}, + {NAME("utilization"), CHILD(named, experimental_utilization)}, + {NAME("arenas"), CHILD(indexed, experimental_arenas)} +}; + static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, {NAME("epoch"), CTL(epoch)}, @@ -548,7 +631,8 @@ static const ctl_named_node_t root_node[] = { {NAME("arena"), CHILD(indexed, arena)}, {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)}, - {NAME("stats"), CHILD(named, stats)} + {NAME("stats"), CHILD(named, stats)}, + {NAME("experimental"), CHILD(named, experimental)} }; static const ctl_named_node_t super_root_node[] = { {NAME(""), CHILD(named, root)} @@ -696,10 +780,14 @@ ctl_arena_clear(ctl_arena_t *ctl_arena) { ctl_arena->astats->nmalloc_small = 0; ctl_arena->astats->ndalloc_small = 0; ctl_arena->astats->nrequests_small = 0; - memset(ctl_arena->astats->bstats, 0, NBINS * + ctl_arena->astats->nfills_small = 0; + ctl_arena->astats->nflushes_small = 0; + memset(ctl_arena->astats->bstats, 0, SC_NBINS * sizeof(bin_stats_t)); - memset(ctl_arena->astats->lstats, 0, (NSIZES - NBINS) * + memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) * sizeof(arena_stats_large_t)); + memset(ctl_arena->astats->estats, 0, SC_NPSIZES * + sizeof(arena_stats_extents_t)); } } @@ -713,9 +801,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, &ctl_arena->pdirty, &ctl_arena->pmuzzy, &ctl_arena->astats->astats, ctl_arena->astats->bstats, - ctl_arena->astats->lstats); + ctl_arena->astats->lstats, ctl_arena->astats->estats); - for (i = 0; i < NBINS; i++) { + for (i = 0; i < SC_NBINS; i++) { ctl_arena->astats->allocated_small += ctl_arena->astats->bstats[i].curregs * sz_index2size(i); @@ -725,6 +813,10 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { ctl_arena->astats->bstats[i].ndalloc; ctl_arena->astats->nrequests_small += ctl_arena->astats->bstats[i].nrequests; + ctl_arena->astats->nfills_small += + ctl_arena->astats->bstats[i].nfills; + ctl_arena->astats->nflushes_small += + ctl_arena->astats->bstats[i].nflushes; } } else { arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, @@ -760,6 +852,8 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, &astats->astats.mapped); accum_atomic_zu(&sdstats->astats.retained, &astats->astats.retained); + accum_atomic_zu(&sdstats->astats.extent_avail, + &astats->astats.extent_avail); } ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, @@ -805,6 +899,8 @@ MUTEX_PROF_ARENA_MUTEXES sdstats->nmalloc_small += astats->nmalloc_small; sdstats->ndalloc_small += astats->ndalloc_small; sdstats->nrequests_small += astats->nrequests_small; + sdstats->nfills_small += astats->nfills_small; + sdstats->nflushes_small += astats->nflushes_small; if (!destroyed) { accum_atomic_zu(&sdstats->astats.allocated_large, @@ -819,6 +915,8 @@ MUTEX_PROF_ARENA_MUTEXES &astats->astats.ndalloc_large); ctl_accum_arena_stats_u64(&sdstats->astats.nrequests_large, &astats->astats.nrequests_large); + accum_atomic_zu(&sdstats->astats.abandoned_vm, + &astats->astats.abandoned_vm); accum_atomic_zu(&sdstats->astats.tcache_bytes, &astats->astats.tcache_bytes); @@ -827,7 +925,8 @@ MUTEX_PROF_ARENA_MUTEXES sdstats->astats.uptime = astats->astats.uptime; } - for (i = 0; i < NBINS; i++) { + /* Merge bin stats. */ + for (i = 0; i < SC_NBINS; i++) { sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; sdstats->bstats[i].nrequests += @@ -846,14 +945,18 @@ MUTEX_PROF_ARENA_MUTEXES if (!destroyed) { sdstats->bstats[i].curslabs += astats->bstats[i].curslabs; + sdstats->bstats[i].nonfull_slabs += + astats->bstats[i].nonfull_slabs; } else { assert(astats->bstats[i].curslabs == 0); + assert(astats->bstats[i].nonfull_slabs == 0); } malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data, &astats->bstats[i].mutex_data); } - for (i = 0; i < NSIZES - NBINS; i++) { + /* Merge stats for large allocations. */ + for (i = 0; i < SC_NSIZES - SC_NBINS; i++) { ctl_accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, &astats->lstats[i].nmalloc); ctl_accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, @@ -867,6 +970,22 @@ MUTEX_PROF_ARENA_MUTEXES assert(astats->lstats[i].curlextents == 0); } } + + /* Merge extents stats. */ + for (i = 0; i < SC_NPSIZES; i++) { + accum_atomic_zu(&sdstats->estats[i].ndirty, + &astats->estats[i].ndirty); + accum_atomic_zu(&sdstats->estats[i].nmuzzy, + &astats->estats[i].nmuzzy); + accum_atomic_zu(&sdstats->estats[i].nretained, + &astats->estats[i].nretained); + accum_atomic_zu(&sdstats->estats[i].dirty_bytes, + &astats->estats[i].dirty_bytes); + accum_atomic_zu(&sdstats->estats[i].muzzy_bytes, + &astats->estats[i].muzzy_bytes); + accum_atomic_zu(&sdstats->estats[i].retained_bytes, + &astats->estats[i].retained_bytes); + } } } @@ -1378,8 +1497,8 @@ label_return: \ #define CTL_RO_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1421,8 +1540,8 @@ label_return: \ */ #define CTL_RO_NL_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1440,8 +1559,8 @@ label_return: \ #define CTL_RO_NL_GEN(n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1475,8 +1594,8 @@ label_return: \ #define CTL_RO_CONFIG_GEN(n, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1494,8 +1613,8 @@ label_return: \ CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) static int -epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; UNUSED uint64_t newval; @@ -1513,8 +1632,9 @@ label_return: } static int -background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +background_thread_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { int ret; bool oldval; @@ -1544,13 +1664,6 @@ background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, background_thread_enabled_set(tsd_tsdn(tsd), newval); if (newval) { - if (!can_enable_background_thread) { - malloc_printf(": Error in dlsym(" - "RTLD_NEXT, \"pthread_create\"). Cannot " - "enable background_thread\n"); - ret = EFAULT; - goto label_return; - } if (background_threads_enable(tsd)) { ret = EFAULT; goto label_return; @@ -1571,8 +1684,9 @@ label_return: } static int -max_background_threads_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +max_background_threads_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; size_t oldval; @@ -1605,13 +1719,6 @@ max_background_threads_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, } if (background_thread_enabled()) { - if (!can_enable_background_thread) { - malloc_printf(": Error in dlsym(" - "RTLD_NEXT, \"pthread_create\"). Cannot " - "enable background_thread\n"); - ret = EFAULT; - goto label_return; - } background_thread_enabled_set(tsd_tsdn(tsd), false); if (background_threads_disable(tsd)) { ret = EFAULT; @@ -1642,6 +1749,7 @@ CTL_RO_CONFIG_GEN(config_debug, bool) CTL_RO_CONFIG_GEN(config_fill, bool) CTL_RO_CONFIG_GEN(config_lazy_lock, bool) CTL_RO_CONFIG_GEN(config_malloc_conf, const char *) +CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool) CTL_RO_CONFIG_GEN(config_prof, bool) CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) @@ -1653,6 +1761,7 @@ CTL_RO_CONFIG_GEN(config_xmalloc, bool) CTL_RO_NL_GEN(opt_abort, opt_abort, bool) CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) +CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *) CTL_RO_NL_GEN(opt_retain, opt_retain, bool) @@ -1660,6 +1769,7 @@ CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], const char *) +CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t) CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t) CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) @@ -1690,8 +1800,8 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) /******************************************************************************/ static int -thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; arena_t *oldarena; unsigned newind, oldind; @@ -1755,8 +1865,9 @@ CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, tsd_thread_deallocatedp_get, uint64_t *) static int -thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -1776,8 +1887,9 @@ label_return: } static int -thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; if (!tcache_available(tsd)) { @@ -1796,8 +1908,9 @@ label_return: } static int -thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; if (!config_prof) { @@ -1827,8 +1940,9 @@ label_return: } static int -thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -1857,8 +1971,8 @@ label_return: /******************************************************************************/ static int -tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1875,8 +1989,8 @@ label_return: } static int -tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1895,8 +2009,8 @@ label_return: } static int -tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -2044,9 +2158,8 @@ arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); if (background_thread_enabled()) { - unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - &background_thread_info[ind]; + background_thread_info_get(arena_ind); assert(info->state == background_thread_started); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_paused; @@ -2059,9 +2172,8 @@ static void arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { if (background_thread_enabled()) { - unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - &background_thread_info[ind]; + background_thread_info_get(arena_ind); assert(info->state == background_thread_paused); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_started; @@ -2217,6 +2329,17 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, ret = EINVAL; goto label_return; } + if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) { + /* + * By default the huge arena purges eagerly. If it is + * set to non-zero decay time afterwards, background + * thread might be needed. + */ + if (background_thread_create(tsd, arena_ind)) { + ret = EFAULT; + goto label_return; + } + } if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp)) { @@ -2300,8 +2423,9 @@ label_return: } static int -arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; unsigned arena_ind; arena_t *arena; @@ -2336,7 +2460,8 @@ label_return: } static const ctl_named_node_t * -arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { +arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, + size_t i) { const ctl_named_node_t *ret; malloc_mutex_lock(tsdn, &ctl_mtx); @@ -2361,8 +2486,8 @@ label_return: /******************************************************************************/ static int -arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned narenas; @@ -2382,8 +2507,9 @@ label_return: } static int -arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { +arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen, bool dirty) { int ret; if (oldp != NULL && oldlenp != NULL) { @@ -2425,34 +2551,36 @@ arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) -CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) +CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) static const ctl_named_node_t * -arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { - if (i > NBINS) { +arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { + if (i > SC_NBINS) { return NULL; } return super_arenas_bin_i_node; } -CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned) -CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(NBINS+(szind_t)mib[2]), +CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned) +CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]), size_t) static const ctl_named_node_t * -arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t i) { - if (i > NSIZES - NBINS) { +arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { + if (i > SC_NSIZES - SC_NBINS) { return NULL; } return super_arenas_lextent_i_node; } static int -arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; extent_hooks_t *extent_hooks; unsigned arena_ind; @@ -2474,8 +2602,9 @@ label_return: } static int -arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; unsigned arena_ind; void *ptr; @@ -2506,8 +2635,9 @@ label_return: /******************************************************************************/ static int -prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -2533,8 +2663,8 @@ label_return: } static int -prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2559,8 +2689,8 @@ label_return: } static int -prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; const char *filename = NULL; @@ -2582,8 +2712,8 @@ label_return: } static int -prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2608,8 +2738,8 @@ label_return: } static int -prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; size_t lg_sample = lg_prof_sample; @@ -2633,6 +2763,44 @@ label_return: CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) +static int +prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + const char *filename = NULL; + + if (!config_prof) { + return ENOENT; + } + + WRITEONLY(); + WRITE(filename, const char *); + + if (prof_log_start(tsd_tsdn(tsd), filename)) { + ret = EFAULT; + goto label_return; + } + + ret = 0; +label_return: + return ret; +} + +static int +prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + if (!config_prof) { + return ENOENT; + } + + if (prof_log_stop(tsd_tsdn(tsd))) { + return EFAULT; + } + + return 0; +} + /******************************************************************************/ CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) @@ -2667,6 +2835,10 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, CTL_RO_CGEN(config_stats, stats_arenas_i_retained, atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, + atomic_load_zu(&arenas_i(mib[2])->astats->astats.extent_avail, + ATOMIC_RELAXED), + size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, ctl_arena_stats_read_u64( @@ -2703,6 +2875,9 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, CTL_RO_CGEN(config_stats, stats_arenas_i_resident, atomic_load_zu(&arenas_i(mib[2])->astats->astats.resident, ATOMIC_RELAXED), size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm, + atomic_load_zu(&arenas_i(mib[2])->astats->astats.abandoned_vm, + ATOMIC_RELAXED), size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, arenas_i(mib[2])->astats->allocated_small, size_t) @@ -2712,6 +2887,10 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, arenas_i(mib[2])->astats->ndalloc_small, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, arenas_i(mib[2])->astats->nrequests_small, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_small_nfills, + arenas_i(mib[2])->astats->nfills_small, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_small_nflushes, + arenas_i(mib[2])->astats->nflushes_small, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, atomic_load_zu(&arenas_i(mib[2])->astats->astats.allocated_large, ATOMIC_RELAXED), size_t) @@ -2721,12 +2900,19 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, ctl_arena_stats_read_u64( &arenas_i(mib[2])->astats->astats.ndalloc_large), uint64_t) -/* - * Note: "nmalloc" here instead of "nrequests" in the read. This is intentional. - */ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, ctl_arena_stats_read_u64( - &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) /* Intentional. */ + &arenas_i(mib[2])->astats->astats.nrequests_large), uint64_t) +/* + * Note: "nmalloc_large" here instead of "nfills" in the read. This is + * intentional (large has no batch fill). + */ +CTL_RO_CGEN(config_stats, stats_arenas_i_large_nfills, + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes, + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.nflushes_large), uint64_t) /* Lock profiling related APIs below. */ #define RO_MUTEX_CTL_GEN(n, l) \ @@ -2765,8 +2951,9 @@ RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, /* Resets all mutex stats, including global, arena and bin mutexes. */ static int -stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { if (!config_stats) { return ENOENT; } @@ -2806,9 +2993,11 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, MUTEX_PROF_RESET(arena->tcache_ql_mtx); MUTEX_PROF_RESET(arena->base->mtx); - for (szind_t i = 0; i < NBINS; i++) { - bin_t *bin = &arena->bins[i]; - MUTEX_PROF_RESET(bin->lock); + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_t *bin = &arena->bins[i].bin_shards[j]; + MUTEX_PROF_RESET(bin->lock); + } } } #undef MUTEX_PROF_RESET @@ -2833,11 +3022,13 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs, arenas_i(mib[2])->astats->bstats[mib[4]].reslabs, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs, + arenas_i(mib[2])->astats->bstats[mib[4]].nonfull_slabs, size_t) static const ctl_named_node_t * -stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t j) { - if (j > NBINS) { +stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j > SC_NBINS) { return NULL; } return super_stats_arenas_i_bins_j_node; @@ -2856,22 +3047,65 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) static const ctl_named_node_t * -stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t j) { - if (j > NSIZES - NBINS) { +stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j > SC_NSIZES - SC_NBINS) { return NULL; } return super_stats_arenas_i_lextents_j_node; } +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].ndirty, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].nretained, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, + ATOMIC_RELAXED), size_t); + static const ctl_named_node_t * -stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { +stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j >= SC_NPSIZES) { + return NULL; + } + return super_stats_arenas_i_extents_j_node; +} + +static bool +ctl_arenas_i_verify(size_t i) { + size_t a = arenas_i2a_impl(i, true, true); + if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { + return true; + } + + return false; +} + +static const ctl_named_node_t * +stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { const ctl_named_node_t *ret; - size_t a; malloc_mutex_lock(tsdn, &ctl_mtx); - a = arenas_i2a_impl(i, true, true); - if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { + if (ctl_arenas_i_verify(i)) { ret = NULL; goto label_return; } @@ -2881,3 +3115,321 @@ label_return: malloc_mutex_unlock(tsdn, &ctl_mtx); return ret; } + +static int +experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + if (oldp == NULL || oldlenp == NULL|| newp == NULL) { + ret = EINVAL; + goto label_return; + } + /* + * Note: this is a *private* struct. This is an experimental interface; + * forcing the user to know the jemalloc internals well enough to + * extract the ABI hopefully ensures nobody gets too comfortable with + * this API, which can change at a moment's notice. + */ + hooks_t hooks; + WRITE(hooks, hooks_t); + void *handle = hook_install(tsd_tsdn(tsd), &hooks); + if (handle == NULL) { + ret = EAGAIN; + goto label_return; + } + READ(handle, void *); + + ret = 0; +label_return: + return ret; +} + +static int +experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + WRITEONLY(); + void *handle = NULL; + WRITE(handle, void *); + if (handle == NULL) { + ret = EINVAL; + goto label_return; + } + hook_remove(tsd_tsdn(tsd), handle); + ret = 0; +label_return: + return ret; +} + +/* + * Output six memory utilization entries for an input pointer, the first one of + * type (void *) and the remaining five of type size_t, describing the following + * (in the same order): + * + * (a) memory address of the extent a potential reallocation would go into, + * == the five fields below describe about the extent the pointer resides in == + * (b) number of free regions in the extent, + * (c) number of regions in the extent, + * (d) size of the extent in terms of bytes, + * (e) total number of free regions in the bin the extent belongs to, and + * (f) total number of regions in the bin the extent belongs to. + * + * Note that "(e)" and "(f)" are only available when stats are enabled; + * otherwise their values are undefined. + * + * This API is mainly intended for small class allocations, where extents are + * used as slab. + * + * In case of large class allocations, "(a)" will be NULL, and "(e)" and "(f)" + * will be zero (if stats are enabled; otherwise undefined). The other three + * fields will be properly set though the values are trivial: "(b)" will be 0, + * "(c)" will be 1, and "(d)" will be the usable size. + * + * The input pointer and size are respectively passed in by newp and newlen, + * and the output fields and size are respectively oldp and *oldlenp. + * + * It can be beneficial to define the following macros to make it easier to + * access the output: + * + * #define SLABCUR_READ(out) (*(void **)out) + * #define COUNTS(out) ((size_t *)((void **)out + 1)) + * #define NFREE_READ(out) COUNTS(out)[0] + * #define NREGS_READ(out) COUNTS(out)[1] + * #define SIZE_READ(out) COUNTS(out)[2] + * #define BIN_NFREE_READ(out) COUNTS(out)[3] + * #define BIN_NREGS_READ(out) COUNTS(out)[4] + * + * and then write e.g. NFREE_READ(oldp) to fetch the output. See the unit test + * test_query in test/unit/extent_util.c for an example. + * + * For a typical defragmentation workflow making use of this API for + * understanding the fragmentation level, please refer to the comment for + * experimental_utilization_batch_query_ctl. + * + * It's up to the application how to determine the significance of + * fragmentation relying on the outputs returned. Possible choices are: + * + * (a) if extent utilization ratio is below certain threshold, + * (b) if extent memory consumption is above certain threshold, + * (c) if extent utilization ratio is significantly below bin utilization ratio, + * (d) if input pointer deviates a lot from potential reallocation address, or + * (e) some selection/combination of the above. + * + * The caller needs to make sure that the input/output arguments are valid, + * in particular, that the size of the output is correct, i.e.: + * + * *oldlenp = sizeof(void *) + sizeof(size_t) * 5 + * + * Otherwise, the function immediately returns EINVAL without touching anything. + * + * In the rare case where there's no associated extent found for the input + * pointer, the function zeros out all output fields and return. Please refer + * to the comment for experimental_utilization_batch_query_ctl to understand the + * motivation from C++. + */ +static int +experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + assert(sizeof(extent_util_stats_verbose_t) + == sizeof(void *) + sizeof(size_t) * 5); + + if (oldp == NULL || oldlenp == NULL + || *oldlenp != sizeof(extent_util_stats_verbose_t) + || newp == NULL) { + ret = EINVAL; + goto label_return; + } + + void *ptr = NULL; + WRITE(ptr, void *); + extent_util_stats_verbose_t *util_stats + = (extent_util_stats_verbose_t *)oldp; + extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr, + &util_stats->nfree, &util_stats->nregs, &util_stats->size, + &util_stats->bin_nfree, &util_stats->bin_nregs, + &util_stats->slabcur_addr); + ret = 0; + +label_return: + return ret; +} + +/* + * Given an input array of pointers, output three memory utilization entries of + * type size_t for each input pointer about the extent it resides in: + * + * (a) number of free regions in the extent, + * (b) number of regions in the extent, and + * (c) size of the extent in terms of bytes. + * + * This API is mainly intended for small class allocations, where extents are + * used as slab. In case of large class allocations, the outputs are trivial: + * "(a)" will be 0, "(b)" will be 1, and "(c)" will be the usable size. + * + * Note that multiple input pointers may reside on a same extent so the output + * fields may contain duplicates. + * + * The format of the input/output looks like: + * + * input[0]: 1st_pointer_to_query | output[0]: 1st_extent_n_free_regions + * | output[1]: 1st_extent_n_regions + * | output[2]: 1st_extent_size + * input[1]: 2nd_pointer_to_query | output[3]: 2nd_extent_n_free_regions + * | output[4]: 2nd_extent_n_regions + * | output[5]: 2nd_extent_size + * ... | ... + * + * The input array and size are respectively passed in by newp and newlen, and + * the output array and size are respectively oldp and *oldlenp. + * + * It can be beneficial to define the following macros to make it easier to + * access the output: + * + * #define NFREE_READ(out, i) out[(i) * 3] + * #define NREGS_READ(out, i) out[(i) * 3 + 1] + * #define SIZE_READ(out, i) out[(i) * 3 + 2] + * + * and then write e.g. NFREE_READ(oldp, i) to fetch the output. See the unit + * test test_batch in test/unit/extent_util.c for a concrete example. + * + * A typical workflow would be composed of the following steps: + * + * (1) flush tcache: mallctl("thread.tcache.flush", ...) + * (2) initialize input array of pointers to query fragmentation + * (3) allocate output array to hold utilization statistics + * (4) query utilization: mallctl("experimental.utilization.batch_query", ...) + * (5) (optional) decide if it's worthwhile to defragment; otherwise stop here + * (6) disable tcache: mallctl("thread.tcache.enabled", ...) + * (7) defragment allocations with significant fragmentation, e.g.: + * for each allocation { + * if it's fragmented { + * malloc(...); + * memcpy(...); + * free(...); + * } + * } + * (8) enable tcache: mallctl("thread.tcache.enabled", ...) + * + * The application can determine the significance of fragmentation themselves + * relying on the statistics returned, both at the overall level i.e. step "(5)" + * and at individual allocation level i.e. within step "(7)". Possible choices + * are: + * + * (a) whether memory utilization ratio is below certain threshold, + * (b) whether memory consumption is above certain threshold, or + * (c) some combination of the two. + * + * The caller needs to make sure that the input/output arrays are valid and + * their sizes are proper as well as matched, meaning: + * + * (a) newlen = n_pointers * sizeof(const void *) + * (b) *oldlenp = n_pointers * sizeof(size_t) * 3 + * (c) n_pointers > 0 + * + * Otherwise, the function immediately returns EINVAL without touching anything. + * + * In the rare case where there's no associated extent found for some pointers, + * rather than immediately terminating the computation and raising an error, + * the function simply zeros out the corresponding output fields and continues + * the computation until all input pointers are handled. The motivations of + * such a design are as follows: + * + * (a) The function always either processes nothing or processes everything, and + * never leaves the output half touched and half untouched. + * + * (b) It facilitates usage needs especially common in C++. A vast variety of + * C++ objects are instantiated with multiple dynamic memory allocations. For + * example, std::string and std::vector typically use at least two allocations, + * one for the metadata and one for the actual content. Other types may use + * even more allocations. When inquiring about utilization statistics, the + * caller often wants to examine into all such allocations, especially internal + * one(s), rather than just the topmost one. The issue comes when some + * implementations do certain optimizations to reduce/aggregate some internal + * allocations, e.g. putting short strings directly into the metadata, and such + * decisions are not known to the caller. Therefore, we permit pointers to + * memory usages that may not be returned by previous malloc calls, and we + * provide the caller a convenient way to identify such cases. + */ +static int +experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + assert(sizeof(extent_util_stats_t) == sizeof(size_t) * 3); + + const size_t len = newlen / sizeof(const void *); + if (oldp == NULL || oldlenp == NULL || newp == NULL || newlen == 0 + || newlen != len * sizeof(const void *) + || *oldlenp != len * sizeof(extent_util_stats_t)) { + ret = EINVAL; + goto label_return; + } + + void **ptrs = (void **)newp; + extent_util_stats_t *util_stats = (extent_util_stats_t *)oldp; + size_t i; + for (i = 0; i < len; ++i) { + extent_util_stats_get(tsd_tsdn(tsd), ptrs[i], + &util_stats[i].nfree, &util_stats[i].nregs, + &util_stats[i].size); + } + ret = 0; + +label_return: + return ret; +} + +static const ctl_named_node_t * +experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { + const ctl_named_node_t *ret; + + malloc_mutex_lock(tsdn, &ctl_mtx); + if (ctl_arenas_i_verify(i)) { + ret = NULL; + goto label_return; + } + ret = super_experimental_arenas_i_node; +label_return: + malloc_mutex_unlock(tsdn, &ctl_mtx); + return ret; +} + +static int +experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + if (!config_stats) { + return ENOENT; + } + if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(size_t *)) { + return EINVAL; + } + + unsigned arena_ind; + arena_t *arena; + int ret; + size_t *pactivep; + + malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); + READONLY(); + MIB_UNSIGNED(arena_ind, 2); + if (arena_ind < narenas_total_get() && (arena = + arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { +#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \ + defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER) + /* Expose the underlying counter for fast read. */ + pactivep = (size_t *)&(arena->nactive.repr); + READ(pactivep, size_t *); + ret = 0; +#else + ret = EFAULT; +#endif + } else { + ret = EFAULT; + } +label_return: + malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); + return ret; +} diff --git a/deps/jemalloc/src/extent.c b/deps/jemalloc/src/extent.c index 09d6d7718..9237f903d 100644 --- a/deps/jemalloc/src/extent.c +++ b/deps/jemalloc/src/extent.c @@ -20,7 +20,7 @@ mutex_pool_t extent_mutex_pool; size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; static const bitmap_info_t extents_bitmap_info = - BITMAP_INFO_INITIALIZER(NPSIZES+1); + BITMAP_INFO_INITIALIZER(SC_NPSIZES+1); static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, @@ -50,20 +50,16 @@ static bool extent_purge_forced_default(extent_hooks_t *extent_hooks, static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length, bool growing_retained); -#ifdef JEMALLOC_MAPS_COALESCE static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind); -#endif static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained); -#ifdef JEMALLOC_MAPS_COALESCE static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind); -#endif static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, bool growing_retained); @@ -88,11 +84,9 @@ const extent_hooks_t extent_hooks_default = { , NULL #endif -#ifdef JEMALLOC_MAPS_COALESCE , extent_split_default, extent_merge_default -#endif }; /* Used exclusively for gdump triggering. */ @@ -119,9 +113,13 @@ static void extent_record(tsdn_t *tsdn, arena_t *arena, /******************************************************************************/ -ph_gen(UNUSED, extent_avail_, extent_tree_t, extent_t, ph_link, +#define ATTR_NONE /* does nothing */ + +ph_gen(ATTR_NONE, extent_avail_, extent_tree_t, extent_t, ph_link, extent_esnead_comp) +#undef ATTR_NONE + typedef enum { lock_result_success, lock_result_failure, @@ -130,13 +128,16 @@ typedef enum { static lock_result_t extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, - extent_t **result) { + extent_t **result, bool inactive_only) { extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree, elm, true); - if (extent1 == NULL) { + /* Slab implies active extents and should be skipped. */ + if (extent1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn, + &extents_rtree, elm, true))) { return lock_result_no_extent; } + /* * It's possible that the extent changed out from under us, and with it * the leaf->extent mapping. We have to recheck while holding the lock. @@ -159,7 +160,8 @@ extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, * address, and NULL otherwise. */ static extent_t * -extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { +extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr, + bool inactive_only) { extent_t *ret = NULL; rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)addr, false, false); @@ -168,7 +170,8 @@ extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { } lock_result_t lock_result; do { - lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret); + lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret, + inactive_only); } while (lock_result == lock_result_failure); return ret; } @@ -182,6 +185,7 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) { return base_alloc_extent(tsdn, arena->base); } extent_avail_remove(&arena->extent_avail, extent); + atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); return extent; } @@ -190,6 +194,7 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); extent_avail_insert(&arena->extent_avail, extent); + atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); } @@ -255,7 +260,7 @@ extent_size_quantize_ceil(size_t size) { size_t ret; assert(size > 0); - assert(size - sz_large_pad <= LARGE_MAXCLASS); + assert(size - sz_large_pad <= SC_LARGE_MAXCLASS); assert((size & PAGE_MASK) == 0); ret = extent_size_quantize_floor(size); @@ -284,7 +289,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, malloc_mutex_rank_exclusive)) { return true; } - for (unsigned i = 0; i < NPSIZES+1; i++) { + for (unsigned i = 0; i < SC_NPSIZES + 1; i++) { extent_heap_new(&extents->heaps[i]); } bitmap_init(extents->bitmap, &extents_bitmap_info, true); @@ -305,6 +310,32 @@ extents_npages_get(extents_t *extents) { return atomic_load_zu(&extents->npages, ATOMIC_RELAXED); } +size_t +extents_nextents_get(extents_t *extents, pszind_t pind) { + return atomic_load_zu(&extents->nextents[pind], ATOMIC_RELAXED); +} + +size_t +extents_nbytes_get(extents_t *extents, pszind_t pind) { + return atomic_load_zu(&extents->nbytes[pind], ATOMIC_RELAXED); +} + +static void +extents_stats_add(extents_t *extent, pszind_t pind, size_t sz) { + size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nextents[pind], cur + 1, ATOMIC_RELAXED); + cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nbytes[pind], cur + sz, ATOMIC_RELAXED); +} + +static void +extents_stats_sub(extents_t *extent, pszind_t pind, size_t sz) { + size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nextents[pind], cur - 1, ATOMIC_RELAXED); + cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nbytes[pind], cur - sz, ATOMIC_RELAXED); +} + static void extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { malloc_mutex_assert_owner(tsdn, &extents->mtx); @@ -318,6 +349,11 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { (size_t)pind); } extent_heap_insert(&extents->heaps[pind], extent); + + if (config_stats) { + extents_stats_add(extents, pind, size); + } + extent_list_append(&extents->lru, extent); size_t npages = size >> LG_PAGE; /* @@ -340,6 +376,11 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { size_t psz = extent_size_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); extent_heap_remove(&extents->heaps[pind], extent); + + if (config_stats) { + extents_stats_sub(extents, pind, size); + } + if (extent_heap_empty(&extents->heaps[pind])) { bitmap_set(extents->bitmap, &extents_bitmap_info, (size_t)pind); @@ -371,7 +412,7 @@ extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, &extents_bitmap_info, (size_t)pind); i < pind_max; i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)i+1)) { - assert(i < NPSIZES); + assert(i < SC_NPSIZES); assert(!extent_heap_empty(&extents->heaps[i])); extent_t *extent = extent_heap_first(&extents->heaps[i]); uintptr_t base = (uintptr_t)extent_base_get(extent); @@ -394,30 +435,6 @@ extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, return NULL; } -/* Do any-best-fit extent selection, i.e. select any extent that best fits. */ -static extent_t * -extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, - size_t size) { - pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); - pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, - (size_t)pind); - if (i < NPSIZES+1) { - /* - * In order to reduce fragmentation, avoid reusing and splitting - * large extents for much smaller sizes. - */ - if ((sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) { - return NULL; - } - assert(!extent_heap_empty(&extents->heaps[i])); - extent_t *extent = extent_heap_first(&extents->heaps[i]); - assert(extent_size_get(extent) >= size); - return extent; - } - - return NULL; -} - /* * Do first-fit extent selection, i.e. select the oldest/lowest extent that is * large enough. @@ -428,30 +445,49 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, extent_t *ret = NULL; pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); + + if (!maps_coalesce && !opt_retain) { + /* + * No split / merge allowed (Windows w/o retain). Try exact fit + * only. + */ + return extent_heap_empty(&extents->heaps[pind]) ? NULL : + extent_heap_first(&extents->heaps[pind]); + } + for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, - &extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i = - (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, + &extents_bitmap_info, (size_t)pind); + i < SC_NPSIZES + 1; + i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)i+1)) { assert(!extent_heap_empty(&extents->heaps[i])); extent_t *extent = extent_heap_first(&extents->heaps[i]); assert(extent_size_get(extent) >= size); + /* + * In order to reduce fragmentation, avoid reusing and splitting + * large extents for much smaller sizes. + * + * Only do check for dirty extents (delay_coalesce). + */ + if (extents->delay_coalesce && + (sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) { + break; + } if (ret == NULL || extent_snad_comp(extent, ret) < 0) { ret = extent; } - if (i == NPSIZES) { + if (i == SC_NPSIZES) { break; } - assert(i < NPSIZES); + assert(i < SC_NPSIZES); } return ret; } /* - * Do {best,first}-fit extent selection, where the selection policy choice is - * based on extents->delay_coalesce. Best-fit selection requires less - * searching, but its layout policy is less stable and may cause higher virtual - * memory fragmentation as a side effect. + * Do first-fit extent selection, where the selection policy choice is + * based on extents->delay_coalesce. */ static extent_t * extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, @@ -464,8 +500,7 @@ extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, return NULL; } - extent_t *extent = extents->delay_coalesce ? - extents_best_fit_locked(tsdn, arena, extents, max_size) : + extent_t *extent = extents_first_fit_locked(tsdn, arena, extents, max_size); if (alignment > PAGE && extent == NULL) { @@ -592,16 +627,24 @@ label_return: return extent; } +/* + * This can only happen when we fail to allocate a new extent struct (which + * indicates OOM), e.g. when trying to split an existing extent. + */ static void -extents_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, +extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent, bool growing_retained) { + size_t sz = extent_size_get(extent); + if (config_stats) { + arena_stats_accum_zu(&arena->stats.abandoned_vm, sz); + } /* * Leak extent after making sure its pages have already been purged, so * that this is only a virtual memory leak. */ if (extents_state_get(extents) == extent_state_dirty) { if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, - extent, 0, extent_size_get(extent), growing_retained)) { + extent, 0, sz, growing_retained)) { extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent, 0, extent_size_get(extent), growing_retained); @@ -748,6 +791,7 @@ extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) { if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true, &elm_a, &elm_b)) { + extent_unlock(tsdn, extent); return true; } @@ -817,7 +861,7 @@ extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool gdump) { extent_lock(tsdn, extent); - extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, NSIZES, false); + extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -874,7 +918,8 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); extent_t *extent; if (new_addr != NULL) { - extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr); + extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr, + false); if (extent != NULL) { /* * We might null-out extent to report an error, but we @@ -958,7 +1003,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, if (leadsize != 0) { *lead = *extent; *extent = extent_split_impl(tsdn, arena, r_extent_hooks, - *lead, leadsize, NSIZES, false, esize + trailsize, szind, + *lead, leadsize, SC_NSIZES, false, esize + trailsize, szind, slab, growing_retained); if (*extent == NULL) { *to_leak = *lead; @@ -970,7 +1015,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, /* Split the trail. */ if (trailsize != 0) { *trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent, - esize, szind, slab, trailsize, NSIZES, false, + esize, szind, slab, trailsize, SC_NSIZES, false, growing_retained); if (*trail == NULL) { *to_leak = *extent; @@ -987,7 +1032,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, * splitting occurred. */ extent_szind_set(*extent, szind); - if (szind != NSIZES) { + if (szind != SC_NSIZES) { rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_addr_get(*extent), szind, slab); if (slab && extent_size_get(*extent) > PAGE) { @@ -1023,6 +1068,17 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, &to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind, growing_retained); + if (!maps_coalesce && result != extent_split_interior_ok + && !opt_retain) { + /* + * Split isn't supported (implies Windows w/o retain). Avoid + * leaking the extents. + */ + assert(to_leak != NULL && lead == NULL && trail == NULL); + extent_deactivate(tsdn, arena, extents, to_leak); + return NULL; + } + if (result == extent_split_interior_ok) { if (lead != NULL) { extent_deactivate(tsdn, arena, extents, lead); @@ -1043,16 +1099,27 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, if (to_leak != NULL) { void *leak = extent_base_get(to_leak); extent_deregister_no_gdump_sub(tsdn, to_leak); - extents_leak(tsdn, arena, r_extent_hooks, extents, + extents_abandon_vm(tsdn, arena, r_extent_hooks, extents, to_leak, growing_retained); - assert(extent_lock_from_addr(tsdn, rtree_ctx, leak) - == NULL); + assert(extent_lock_from_addr(tsdn, rtree_ctx, leak, + false) == NULL); } return NULL; } unreachable(); } +static bool +extent_need_manual_zero(arena_t *arena) { + /* + * Need to manually zero the extent on repopulating if either; 1) non + * default extent hooks installed (in which case the purge semantics may + * change); or 2) transparent huge pages enabled. + */ + return (!arena_has_default_hooks(arena) || + (opt_thp == thp_mode_always)); +} + /* * Tries to satisfy the given allocation request by reusing one of the extents * in the given extents_t. @@ -1092,7 +1159,9 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent, growing_retained); return NULL; } - extent_zeroed_set(extent, true); + if (!extent_need_manual_zero(arena)) { + extent_zeroed_set(extent, true); + } } if (extent_committed_get(extent)) { @@ -1113,14 +1182,16 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (*zero) { void *addr = extent_base_get(extent); - size_t size = extent_size_get(extent); if (!extent_zeroed_get(extent)) { - if (pages_purge_forced(addr, size)) { + size_t size = extent_size_get(extent); + if (extent_need_manual_zero(arena) || + pages_purge_forced(addr, size)) { memset(addr, 0, size); } } else if (config_debug) { size_t *p = (size_t *)(uintptr_t)addr; - for (size_t i = 0; i < size / sizeof(size_t); i++) { + /* Check the first page only. */ + for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { assert(p[i] == 0); } } @@ -1191,7 +1262,7 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, assert(arena != NULL); return extent_alloc_default_impl(tsdn, arena, new_addr, size, - alignment, zero, commit); + ALIGNMENT_CEILING(alignment, PAGE), zero, commit); } static void @@ -1244,11 +1315,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); while (alloc_size < alloc_size_min) { egn_skip++; - if (arena->extent_grow_next + egn_skip == NPSIZES) { + if (arena->extent_grow_next + egn_skip >= + sz_psz2ind(SC_LARGE_MAXCLASS)) { /* Outside legal range. */ goto label_err; } - assert(arena->extent_grow_next + egn_skip < NPSIZES); alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); } @@ -1271,17 +1342,16 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, extent_hook_post_reentrancy(tsdn); } - extent_init(extent, arena, ptr, alloc_size, false, NSIZES, + extent_init(extent, arena, ptr, alloc_size, false, SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed, - committed, true); + committed, true, EXTENT_IS_HEAD); if (ptr == NULL) { extent_dalloc(tsdn, arena, extent); goto label_err; } if (extent_register_no_gdump_add(tsdn, extent)) { - extents_leak(tsdn, arena, r_extent_hooks, - &arena->extents_retained, extent, true); + extent_dalloc(tsdn, arena, extent); goto label_err; } @@ -1328,7 +1398,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, } if (to_leak != NULL) { extent_deregister_no_gdump_sub(tsdn, to_leak); - extents_leak(tsdn, arena, r_extent_hooks, + extents_abandon_vm(tsdn, arena, r_extent_hooks, &arena->extents_retained, to_leak, true); } goto label_err; @@ -1341,7 +1411,9 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, &arena->extents_retained, extent, true); goto label_err; } - extent_zeroed_set(extent, true); + if (!extent_need_manual_zero(arena)) { + extent_zeroed_set(extent, true); + } } /* @@ -1375,7 +1447,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, if (*zero && !extent_zeroed_get(extent)) { void *addr = extent_base_get(extent); size_t size = extent_size_get(extent); - if (pages_purge_forced(addr, size)) { + if (extent_need_manual_zero(arena) || + pages_purge_forced(addr, size)) { memset(addr, 0, size); } } @@ -1425,14 +1498,15 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, return NULL; } void *addr; + size_t palignment = ALIGNMENT_CEILING(alignment, PAGE); if (*r_extent_hooks == &extent_hooks_default) { /* Call directly to propagate tsdn. */ addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, - alignment, zero, commit); + palignment, zero, commit); } else { extent_hook_pre_reentrancy(tsdn, arena); addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, - esize, alignment, zero, commit, arena_ind_get(arena)); + esize, palignment, zero, commit, arena_ind_get(arena)); extent_hook_post_reentrancy(tsdn); } if (addr == NULL) { @@ -1441,13 +1515,12 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, } extent_init(extent, arena, addr, esize, slab, szind, arena_extent_sn_next(arena), extent_state_active, *zero, *commit, - true); + true, EXTENT_NOT_HEAD); if (pad != 0) { extent_addr_randomize(tsdn, extent, alignment); } if (extent_register(tsdn, extent)) { - extents_leak(tsdn, arena, r_extent_hooks, - &arena->extents_retained, extent, false); + extent_dalloc(tsdn, arena, extent); return NULL; } @@ -1524,9 +1597,15 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, } static extent_t * -extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, +extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, - extent_t *extent, bool *coalesced, bool growing_retained) { + extent_t *extent, bool *coalesced, bool growing_retained, + bool inactive_only) { + /* + * We avoid checking / locking inactive neighbors for large size + * classes, since they are eagerly coalesced on deallocation which can + * cause lock contention. + */ /* * Continue attempting to coalesce until failure, to protect against * races with other threads that are thwarted by this one. @@ -1537,7 +1616,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce forward. */ extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx, - extent_past_get(extent)); + extent_past_get(extent), inactive_only); if (next != NULL) { /* * extents->mtx only protects against races for @@ -1563,7 +1642,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce backward. */ extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx, - extent_before_get(extent)); + extent_before_get(extent), inactive_only); if (prev != NULL) { bool can_coalesce = extent_can_coalesce(arena, extents, extent, prev); @@ -1589,6 +1668,22 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, return extent; } +static extent_t * +extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + extent_t *extent, bool *coalesced, bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, + extents, extent, coalesced, growing_retained, false); +} + +static extent_t * +extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + extent_t *extent, bool *coalesced, bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, + extents, extent, coalesced, growing_retained, true); +} + /* * Does the metadata management portions of putting an unused extent into the * given extents_t (coalesces, deregisters slab interiors, the heap operations). @@ -1606,7 +1701,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, malloc_mutex_lock(tsdn, &extents->mtx); extent_hooks_assure_initialized(arena, r_extent_hooks); - extent_szind_set(extent, NSIZES); + extent_szind_set(extent, SC_NSIZES); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -1618,18 +1713,22 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (!extents->delay_coalesce) { extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, extents, extent, NULL, growing_retained); - } else if (extent_size_get(extent) >= LARGE_MINCLASS) { + } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) { + assert(extents == &arena->extents_dirty); /* Always coalesce large extents eagerly. */ bool coalesced; - size_t prev_size; do { - prev_size = extent_size_get(extent); assert(extent_state_get(extent) == extent_state_active); - extent = extent_try_coalesce(tsdn, arena, + extent = extent_try_coalesce_large(tsdn, arena, r_extent_hooks, rtree_ctx, extents, extent, &coalesced, growing_retained); - } while (coalesced && - extent_size_get(extent) >= prev_size + LARGE_MINCLASS); + } while (coalesced); + if (extent_size_get(extent) >= oversize_threshold) { + /* Shortcut to purge the oversize extent eagerly. */ + malloc_mutex_unlock(tsdn, &extents->mtx); + arena_decay_extent(tsdn, arena, r_extent_hooks, extent); + return; + } } extent_deactivate_locked(tsdn, arena, extents, extent); @@ -1644,13 +1743,18 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { WITNESS_RANK_CORE, 0); if (extent_register(tsdn, extent)) { - extents_leak(tsdn, arena, &extent_hooks, - &arena->extents_retained, extent, false); + extent_dalloc(tsdn, arena, extent); return; } extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); } +static bool +extent_may_dalloc(void) { + /* With retain enabled, the default dalloc always fails. */ + return !opt_retain; +} + static bool extent_dalloc_default_impl(void *addr, size_t size) { if (!have_dss || !extent_in_dss(addr)) { @@ -1706,16 +1810,20 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - /* - * Deregister first to avoid a race with other allocating threads, and - * reregister if deallocation fails. - */ - extent_deregister(tsdn, extent); - if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) { - return; + /* Avoid calling the default extent_dalloc unless have to. */ + if (*r_extent_hooks != &extent_hooks_default || extent_may_dalloc()) { + /* + * Deregister first to avoid a race with other allocating + * threads, and reregister if deallocation fails. + */ + extent_deregister(tsdn, extent); + if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, + extent)) { + return; + } + extent_reregister(tsdn, extent); } - extent_reregister(tsdn, extent); if (*r_extent_hooks != &extent_hooks_default) { extent_hook_pre_reentrancy(tsdn, arena); } @@ -1955,13 +2063,20 @@ extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, offset, length, false); } -#ifdef JEMALLOC_MAPS_COALESCE static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { - return !maps_coalesce; + if (!maps_coalesce) { + /* + * Without retain, only whole regions can be purged (required by + * MEM_RELEASE on Windows) -- therefore disallow splitting. See + * comments in extent_head_no_merge(). + */ + return !opt_retain; + } + + return false; } -#endif /* * Accepts the extent to split, and the characteristics of each side of the @@ -1993,7 +2108,8 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + size_a), size_b, slab_b, szind_b, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent), extent_dumpable_get(extent)); + extent_committed_get(extent), extent_dumpable_get(extent), + EXTENT_NOT_HEAD); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -2004,7 +2120,8 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_init(&lead, arena, extent_addr_get(extent), size_a, slab_a, szind_a, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent), extent_dumpable_get(extent)); + extent_committed_get(extent), extent_dumpable_get(extent), + EXTENT_NOT_HEAD); extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false, true, &lead_elm_a, &lead_elm_b); @@ -2062,7 +2179,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, static bool extent_merge_default_impl(void *addr_a, void *addr_b) { - if (!maps_coalesce) { + if (!maps_coalesce && !opt_retain) { return true; } if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { @@ -2072,13 +2189,51 @@ extent_merge_default_impl(void *addr_a, void *addr_b) { return false; } -#ifdef JEMALLOC_MAPS_COALESCE +/* + * Returns true if the given extents can't be merged because of their head bit + * settings. Assumes the second extent has the higher address. + */ +static bool +extent_head_no_merge(extent_t *a, extent_t *b) { + assert(extent_base_get(a) < extent_base_get(b)); + /* + * When coalesce is not always allowed (Windows), only merge extents + * from the same VirtualAlloc region under opt.retain (in which case + * MEM_DECOMMIT is utilized for purging). + */ + if (maps_coalesce) { + return false; + } + if (!opt_retain) { + return true; + } + /* If b is a head extent, disallow the cross-region merge. */ + if (extent_is_head_get(b)) { + /* + * Additionally, sn should not overflow with retain; sanity + * check that different regions have unique sn. + */ + assert(extent_sn_comp(a, b) != 0); + return true; + } + assert(extent_sn_comp(a, b) == 0); + + return false; +} + static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { + if (!maps_coalesce) { + tsdn_t *tsdn = tsdn_fetch(); + extent_t *a = iealloc(tsdn, addr_a); + extent_t *b = iealloc(tsdn, addr_b); + if (extent_head_no_merge(a, b)) { + return true; + } + } return extent_merge_default_impl(addr_a, addr_b); } -#endif static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, @@ -2086,10 +2241,11 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); + assert(extent_base_get(a) < extent_base_get(b)); extent_hooks_assure_initialized(arena, r_extent_hooks); - if ((*r_extent_hooks)->merge == NULL) { + if ((*r_extent_hooks)->merge == NULL || extent_head_no_merge(a, b)) { return true; } @@ -2128,22 +2284,23 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, if (a_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL, - NSIZES, false); + SC_NSIZES, false); } if (b_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL, - NSIZES, false); + SC_NSIZES, false); } else { b_elm_b = b_elm_a; } extent_size_set(a, extent_size_get(a) + extent_size_get(b)); - extent_szind_set(a, NSIZES); + extent_szind_set(a, SC_NSIZES); extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ? extent_sn_get(a) : extent_sn_get(b)); extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b)); - extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES, false); + extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES, + false); extent_unlock2(tsdn, a, b); @@ -2175,3 +2332,72 @@ extent_boot(void) { return false; } + +void +extent_util_stats_get(tsdn_t *tsdn, const void *ptr, + size_t *nfree, size_t *nregs, size_t *size) { + assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL); + + const extent_t *extent = iealloc(tsdn, ptr); + if (unlikely(extent == NULL)) { + *nfree = *nregs = *size = 0; + return; + } + + *size = extent_size_get(extent); + if (!extent_slab_get(extent)) { + *nfree = 0; + *nregs = 1; + } else { + *nfree = extent_nfree_get(extent); + *nregs = bin_infos[extent_szind_get(extent)].nregs; + assert(*nfree <= *nregs); + assert(*nfree * extent_usize_get(extent) <= *size); + } +} + +void +extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr, + size_t *nfree, size_t *nregs, size_t *size, + size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr) { + assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL + && bin_nfree != NULL && bin_nregs != NULL && slabcur_addr != NULL); + + const extent_t *extent = iealloc(tsdn, ptr); + if (unlikely(extent == NULL)) { + *nfree = *nregs = *size = *bin_nfree = *bin_nregs = 0; + *slabcur_addr = NULL; + return; + } + + *size = extent_size_get(extent); + if (!extent_slab_get(extent)) { + *nfree = *bin_nfree = *bin_nregs = 0; + *nregs = 1; + *slabcur_addr = NULL; + return; + } + + *nfree = extent_nfree_get(extent); + const szind_t szind = extent_szind_get(extent); + *nregs = bin_infos[szind].nregs; + assert(*nfree <= *nregs); + assert(*nfree * extent_usize_get(extent) <= *size); + + const arena_t *arena = extent_arena_get(extent); + assert(arena != NULL); + const unsigned binshard = extent_binshard_get(extent); + bin_t *bin = &arena->bins[szind].bin_shards[binshard]; + + malloc_mutex_lock(tsdn, &bin->lock); + if (config_stats) { + *bin_nregs = *nregs * bin->stats.curslabs; + assert(*bin_nregs >= bin->stats.curregs); + *bin_nfree = *bin_nregs - bin->stats.curregs; + } else { + *bin_nfree = *bin_nregs = 0; + } + *slabcur_addr = extent_addr_get(bin->slabcur); + assert(*slabcur_addr != NULL); + malloc_mutex_unlock(tsdn, &bin->lock); +} diff --git a/deps/jemalloc/src/extent_dss.c b/deps/jemalloc/src/extent_dss.c index 2b1ea9caf..858178911 100644 --- a/deps/jemalloc/src/extent_dss.c +++ b/deps/jemalloc/src/extent_dss.c @@ -113,7 +113,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, cassert(have_dss); assert(size > 0); - assert(alignment > 0); + assert(alignment == ALIGNMENT_CEILING(alignment, PAGE)); /* * sbrk() uses a signed increment argument, so take care not to @@ -154,9 +154,10 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, (uintptr_t)gap_addr_page; if (gap_size_page != 0) { extent_init(gap, arena, gap_addr_page, - gap_size_page, false, NSIZES, + gap_size_page, false, SC_NSIZES, arena_extent_sn_next(arena), - extent_state_active, false, true, true); + extent_state_active, false, true, true, + EXTENT_NOT_HEAD); } /* * Compute the address just past the end of the desired @@ -198,9 +199,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, extent_t extent; extent_init(&extent, arena, ret, size, - size, false, NSIZES, + size, false, SC_NSIZES, extent_state_active, false, true, - true); + true, EXTENT_NOT_HEAD); if (extent_purge_forced_wrapper(tsdn, arena, &extent_hooks, &extent, 0, size)) { diff --git a/deps/jemalloc/src/extent_mmap.c b/deps/jemalloc/src/extent_mmap.c index 8d607dc80..17fd1c8f9 100644 --- a/deps/jemalloc/src/extent_mmap.c +++ b/deps/jemalloc/src/extent_mmap.c @@ -21,8 +21,8 @@ bool opt_retain = void * extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { - void *ret = pages_map(new_addr, size, ALIGNMENT_CEILING(alignment, - PAGE), commit); + assert(alignment == ALIGNMENT_CEILING(alignment, PAGE)); + void *ret = pages_map(new_addr, size, alignment, commit); if (ret == NULL) { return NULL; } diff --git a/deps/jemalloc/src/hook.c b/deps/jemalloc/src/hook.c new file mode 100644 index 000000000..9ac703cf9 --- /dev/null +++ b/deps/jemalloc/src/hook.c @@ -0,0 +1,195 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/hook.h" + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/seq.h" + +typedef struct hooks_internal_s hooks_internal_t; +struct hooks_internal_s { + hooks_t hooks; + bool in_use; +}; + +seq_define(hooks_internal_t, hooks) + +static atomic_u_t nhooks = ATOMIC_INIT(0); +static seq_hooks_t hooks[HOOK_MAX]; +static malloc_mutex_t hooks_mu; + +bool +hook_boot() { + return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK, + malloc_mutex_rank_exclusive); +} + +static void * +hook_install_locked(hooks_t *to_install) { + hooks_internal_t hooks_internal; + for (int i = 0; i < HOOK_MAX; i++) { + bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]); + /* We hold mu; no concurrent access. */ + assert(success); + if (!hooks_internal.in_use) { + hooks_internal.hooks = *to_install; + hooks_internal.in_use = true; + seq_store_hooks(&hooks[i], &hooks_internal); + atomic_store_u(&nhooks, + atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1, + ATOMIC_RELAXED); + return &hooks[i]; + } + } + return NULL; +} + +void * +hook_install(tsdn_t *tsdn, hooks_t *to_install) { + malloc_mutex_lock(tsdn, &hooks_mu); + void *ret = hook_install_locked(to_install); + if (ret != NULL) { + tsd_global_slow_inc(tsdn); + } + malloc_mutex_unlock(tsdn, &hooks_mu); + return ret; +} + +static void +hook_remove_locked(seq_hooks_t *to_remove) { + hooks_internal_t hooks_internal; + bool success = seq_try_load_hooks(&hooks_internal, to_remove); + /* We hold mu; no concurrent access. */ + assert(success); + /* Should only remove hooks that were added. */ + assert(hooks_internal.in_use); + hooks_internal.in_use = false; + seq_store_hooks(to_remove, &hooks_internal); + atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1, + ATOMIC_RELAXED); +} + +void +hook_remove(tsdn_t *tsdn, void *opaque) { + if (config_debug) { + char *hooks_begin = (char *)&hooks[0]; + char *hooks_end = (char *)&hooks[HOOK_MAX]; + char *hook = (char *)opaque; + assert(hooks_begin <= hook && hook < hooks_end + && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0); + } + malloc_mutex_lock(tsdn, &hooks_mu); + hook_remove_locked((seq_hooks_t *)opaque); + tsd_global_slow_dec(tsdn); + malloc_mutex_unlock(tsdn, &hooks_mu); +} + +#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \ +for (int for_each_hook_counter = 0; \ + for_each_hook_counter < HOOK_MAX; \ + for_each_hook_counter++) { \ + bool for_each_hook_success = seq_try_load_hooks( \ + (hooks_internal_ptr), &hooks[for_each_hook_counter]); \ + if (!for_each_hook_success) { \ + continue; \ + } \ + if (!(hooks_internal_ptr)->in_use) { \ + continue; \ + } +#define FOR_EACH_HOOK_END \ +} + +static bool * +hook_reentrantp() { + /* + * We prevent user reentrancy within hooks. This is basically just a + * thread-local bool that triggers an early-exit. + * + * We don't fold in_hook into reentrancy. There are two reasons for + * this: + * - Right now, we turn on reentrancy during things like extent hook + * execution. Allocating during extent hooks is not officially + * supported, but we don't want to break it for the time being. These + * sorts of allocations should probably still be hooked, though. + * - If a hook allocates, we may want it to be relatively fast (after + * all, it executes on every allocator operation). Turning on + * reentrancy is a fairly heavyweight mode (disabling tcache, + * redirecting to arena 0, etc.). It's possible we may one day want + * to turn on reentrant mode here, if it proves too difficult to keep + * this working. But that's fairly easy for us to see; OTOH, people + * not using hooks because they're too slow is easy for us to miss. + * + * The tricky part is + * that this code might get invoked even if we don't have access to tsd. + * This function mimics getting a pointer to thread-local data, except + * that it might secretly return a pointer to some global data if we + * know that the caller will take the early-exit path. + * If we return a bool that indicates that we are reentrant, then the + * caller will go down the early exit path, leaving the global + * untouched. + */ + static bool in_hook_global = true; + tsdn_t *tsdn = tsdn_fetch(); + tcache_t *tcache = tsdn_tcachep_get(tsdn); + if (tcache != NULL) { + return &tcache->in_hook; + } + return &in_hook_global; +} + +#define HOOK_PROLOGUE \ + if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \ + return; \ + } \ + bool *in_hook = hook_reentrantp(); \ + if (*in_hook) { \ + return; \ + } \ + *in_hook = true; + +#define HOOK_EPILOGUE \ + *in_hook = false; + +void +hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]) { + HOOK_PROLOGUE + + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_alloc h = hook.hooks.alloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, result, result_raw, args_raw); + } + FOR_EACH_HOOK_END + + HOOK_EPILOGUE +} + +void +hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) { + HOOK_PROLOGUE + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_dalloc h = hook.hooks.dalloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, args_raw); + } + FOR_EACH_HOOK_END + HOOK_EPILOGUE +} + +void +hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) { + HOOK_PROLOGUE + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_expand h = hook.hooks.expand_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, old_usize, new_usize, + result_raw, args_raw); + } + FOR_EACH_HOOK_END + HOOK_EPILOGUE +} diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c index 585645a28..97da1ee72 100644 --- a/deps/jemalloc/src/jemalloc.c +++ b/deps/jemalloc/src/jemalloc.c @@ -7,12 +7,14 @@ #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/log.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/safety_check.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/spin.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" @@ -41,6 +43,8 @@ bool opt_abort_conf = false #endif ; +/* Intentionally default off, even with debug builds. */ +bool opt_confirm_conf = false; const char *opt_junk = #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) "true" @@ -85,8 +89,10 @@ malloc_mutex_t arenas_lock; JEMALLOC_ALIGNED(CACHELINE) atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; static atomic_u_t narenas_total; /* Use narenas_total_*(). */ -static arena_t *a0; /* arenas[0]; read-only after initialization. */ -unsigned narenas_auto; /* Read-only after initialization. */ +/* Below three are read-only after initialization. */ +static arena_t *a0; /* arenas[0]. */ +unsigned narenas_auto; +unsigned manual_arena_base; typedef enum { malloc_init_uninitialized = 3, @@ -326,7 +332,7 @@ arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ arena = arena_get(tsdn, ind, false); if (arena != NULL) { - assert(ind < narenas_auto); + assert(arena_is_auto(arena)); return arena; } @@ -341,12 +347,12 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { if (ind == 0) { return; } - if (have_background_thread) { - bool err; - malloc_mutex_lock(tsdn, &background_thread_lock); - err = background_thread_create(tsdn_tsd(tsdn), ind); - malloc_mutex_unlock(tsdn, &background_thread_lock); - if (err) { + /* + * Avoid creating a new background thread just for the huge arena, which + * purges eagerly by default. + */ + if (have_background_thread && !arena_is_huge(ind)) { + if (background_thread_create(tsdn_tsd(tsdn), ind)) { malloc_printf(": error in background thread " "creation for arena %u. Abort.\n", ind); abort(); @@ -376,6 +382,14 @@ arena_bind(tsd_t *tsd, unsigned ind, bool internal) { tsd_iarena_set(tsd, arena); } else { tsd_arena_set(tsd, arena); + unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1, + ATOMIC_RELAXED); + tsd_binshards_t *bins = tsd_binshardsp_get(tsd); + for (unsigned i = 0; i < SC_NBINS; i++) { + assert(bin_infos[i].n_shards > 0 && + bin_infos[i].n_shards <= BIN_SHARDS_MAX); + bins->binshard[i] = shard % bin_infos[i].n_shards; + } } } @@ -761,6 +775,50 @@ init_opt_stats_print_opts(const char *v, size_t vlen) { assert(opts_len == strlen(opt_stats_print_opts)); } +/* Reads the next size pair in a multi-sized option. */ +static bool +malloc_conf_multi_sizes_next(const char **slab_size_segment_cur, + size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) { + const char *cur = *slab_size_segment_cur; + char *end; + uintmax_t um; + + set_errno(0); + + /* First number, then '-' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != '-') { + return true; + } + *slab_start = (size_t)um; + cur = end + 1; + + /* Second number, then ':' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != ':') { + return true; + } + *slab_end = (size_t)um; + cur = end + 1; + + /* Last number */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0) { + return true; + } + *new_size = (size_t)um; + + /* Consume the separator if there is one. */ + if (*end == '|') { + end++; + } + + *vlen_left -= end - *slab_size_segment_cur; + *slab_size_segment_cur = end; + + return false; +} + static bool malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, char const **v_p, size_t *vlen_p) { @@ -850,6 +908,11 @@ malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, malloc_printf(": %s: %.*s:%.*s\n", msg, (int)klen, k, (int)vlen, v); /* If abort_conf is set, error out after processing all options. */ + const char *experimental = "experimental_"; + if (strncmp(k, experimental, strlen(experimental)) == 0) { + /* However, tolerate experimental features. */ + return; + } had_conf_error = true; } @@ -868,88 +931,141 @@ malloc_slow_flag_init(void) { malloc_slow = (malloc_slow_flags != 0); } +/* Number of sources for initializing malloc_conf */ +#define MALLOC_CONF_NSOURCES 4 + +static const char * +obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) { + if (config_debug) { + static unsigned read_source = 0; + /* + * Each source should only be read once, to minimize # of + * syscalls on init. + */ + assert(read_source++ == which_source); + } + assert(which_source < MALLOC_CONF_NSOURCES); + + const char *ret; + switch (which_source) { + case 0: + ret = config_malloc_conf; + break; + case 1: + if (je_malloc_conf != NULL) { + /* Use options that were compiled into the program. */ + ret = je_malloc_conf; + } else { + /* No configuration specified. */ + ret = NULL; + } + break; + case 2: { + ssize_t linklen = 0; +#ifndef _WIN32 + int saved_errno = errno; + const char *linkname = +# ifdef JEMALLOC_PREFIX + "/etc/"JEMALLOC_PREFIX"malloc.conf" +# else + "/etc/malloc.conf" +# endif + ; + + /* + * Try to use the contents of the "/etc/malloc.conf" symbolic + * link's name. + */ +#ifndef JEMALLOC_READLINKAT + linklen = readlink(linkname, buf, PATH_MAX); +#else + linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX); +#endif + if (linklen == -1) { + /* No configuration specified. */ + linklen = 0; + /* Restore errno. */ + set_errno(saved_errno); + } +#endif + buf[linklen] = '\0'; + ret = buf; + break; + } case 3: { + const char *envname = +#ifdef JEMALLOC_PREFIX + JEMALLOC_CPREFIX"MALLOC_CONF" +#else + "MALLOC_CONF" +#endif + ; + + if ((ret = jemalloc_secure_getenv(envname)) != NULL) { + /* + * Do nothing; opts is already initialized to the value + * of the MALLOC_CONF environment variable. + */ + } else { + /* No configuration specified. */ + ret = NULL; + } + break; + } default: + not_reached(); + ret = NULL; + } + return ret; +} + static void -malloc_conf_init(void) { +malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], + bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES], + char buf[PATH_MAX + 1]) { + static const char *opts_explain[MALLOC_CONF_NSOURCES] = { + "string specified via --with-malloc-conf", + "string pointed to by the global variable malloc_conf", + "\"name\" of the file referenced by the symbolic link named " + "/etc/malloc.conf", + "value of the environment variable MALLOC_CONF" + }; unsigned i; - char buf[PATH_MAX + 1]; const char *opts, *k, *v; size_t klen, vlen; - for (i = 0; i < 4; i++) { + for (i = 0; i < MALLOC_CONF_NSOURCES; i++) { /* Get runtime configuration. */ - switch (i) { - case 0: - opts = config_malloc_conf; - break; - case 1: - if (je_malloc_conf != NULL) { - /* - * Use options that were compiled into the - * program. - */ - opts = je_malloc_conf; - } else { - /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; - } - break; - case 2: { - ssize_t linklen = 0; -#ifndef _WIN32 - int saved_errno = errno; - const char *linkname = -# ifdef JEMALLOC_PREFIX - "/etc/"JEMALLOC_PREFIX"malloc.conf" -# else - "/etc/malloc.conf" -# endif - ; - - /* - * Try to use the contents of the "/etc/malloc.conf" - * symbolic link's name. - */ - linklen = readlink(linkname, buf, sizeof(buf) - 1); - if (linklen == -1) { - /* No configuration specified. */ - linklen = 0; - /* Restore errno. */ - set_errno(saved_errno); - } -#endif - buf[linklen] = '\0'; - opts = buf; - break; - } case 3: { - const char *envname = -#ifdef JEMALLOC_PREFIX - JEMALLOC_CPREFIX"MALLOC_CONF" -#else - "MALLOC_CONF" -#endif - ; - - if ((opts = jemalloc_secure_getenv(envname)) != NULL) { - /* - * Do nothing; opts is already initialized to - * the value of the MALLOC_CONF environment - * variable. - */ - } else { - /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; - } - break; - } default: - not_reached(); - buf[0] = '\0'; - opts = buf; + if (initial_call) { + opts_cache[i] = obtain_malloc_conf(i, buf); + } + opts = opts_cache[i]; + if (!initial_call && opt_confirm_conf) { + malloc_printf( + ": malloc_conf #%u (%s): \"%s\"\n", + i + 1, opts_explain[i], opts != NULL ? opts : ""); + } + if (opts == NULL) { + continue; } while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, &vlen)) { + +#define CONF_ERROR(msg, k, klen, v, vlen) \ + if (!initial_call) { \ + malloc_conf_error( \ + msg, k, klen, v, vlen); \ + cur_opt_valid = false; \ + } +#define CONF_CONTINUE { \ + if (!initial_call && opt_confirm_conf \ + && cur_opt_valid) { \ + malloc_printf(": -- " \ + "Set conf value: %.*s:%.*s" \ + "\n", (int)klen, k, \ + (int)vlen, v); \ + } \ + continue; \ + } #define CONF_MATCH(n) \ (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) #define CONF_MATCH_VALUE(n) \ @@ -961,16 +1077,23 @@ malloc_conf_init(void) { } else if (CONF_MATCH_VALUE("false")) { \ o = false; \ } else { \ - malloc_conf_error( \ - "Invalid conf value", \ + CONF_ERROR("Invalid conf value",\ k, klen, v, vlen); \ } \ - continue; \ + CONF_CONTINUE; \ } -#define CONF_MIN_no(um, min) false -#define CONF_MIN_yes(um, min) ((um) < (min)) -#define CONF_MAX_no(um, max) false -#define CONF_MAX_yes(um, max) ((um) > (max)) + /* + * One of the CONF_MIN macros below expands, in one of the use points, + * to "unsigned integer < 0", which is always false, triggering the + * GCC -Wtype-limits warning, which we disable here and re-enable below. + */ + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS + +#define CONF_DONT_CHECK_MIN(um, min) false +#define CONF_CHECK_MIN(um, min) ((um) < (min)) +#define CONF_DONT_CHECK_MAX(um, max) false +#define CONF_CHECK_MAX(um, max) ((um) > (max)) #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ if (CONF_MATCH(n)) { \ uintmax_t um; \ @@ -980,26 +1103,21 @@ malloc_conf_init(void) { um = malloc_strtoumax(v, &end, 0); \ if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ - malloc_conf_error( \ - "Invalid conf value", \ + CONF_ERROR("Invalid conf value",\ k, klen, v, vlen); \ } else if (clip) { \ - if (CONF_MIN_##check_min(um, \ - (t)(min))) { \ + if (check_min(um, (t)(min))) { \ o = (t)(min); \ } else if ( \ - CONF_MAX_##check_max(um, \ - (t)(max))) { \ + check_max(um, (t)(max))) { \ o = (t)(max); \ } else { \ o = (t)um; \ } \ } else { \ - if (CONF_MIN_##check_min(um, \ - (t)(min)) || \ - CONF_MAX_##check_max(um, \ - (t)(max))) { \ - malloc_conf_error( \ + if (check_min(um, (t)(min)) || \ + check_max(um, (t)(max))) { \ + CONF_ERROR( \ "Out-of-range " \ "conf value", \ k, klen, v, vlen); \ @@ -1007,7 +1125,7 @@ malloc_conf_init(void) { o = (t)um; \ } \ } \ - continue; \ + CONF_CONTINUE; \ } #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ clip) \ @@ -1025,18 +1143,17 @@ malloc_conf_init(void) { l = strtol(v, &end, 0); \ if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ - malloc_conf_error( \ - "Invalid conf value", \ + CONF_ERROR("Invalid conf value",\ k, klen, v, vlen); \ } else if (l < (ssize_t)(min) || l > \ (ssize_t)(max)) { \ - malloc_conf_error( \ + CONF_ERROR( \ "Out-of-range conf value", \ k, klen, v, vlen); \ } else { \ o = l; \ } \ - continue; \ + CONF_CONTINUE; \ } #define CONF_HANDLE_CHAR_P(o, n, d) \ if (CONF_MATCH(n)) { \ @@ -1045,7 +1162,14 @@ malloc_conf_init(void) { sizeof(o)-1; \ strncpy(o, v, cpylen); \ o[cpylen] = '\0'; \ - continue; \ + CONF_CONTINUE; \ + } + + bool cur_opt_valid = true; + + CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf") + if (initial_call) { + continue; } CONF_HANDLE_BOOL(opt_abort, "abort") @@ -1062,10 +1186,10 @@ malloc_conf_init(void) { } } if (!match) { - malloc_conf_error("Invalid conf value", + CONF_ERROR("Invalid conf value", k, klen, v, vlen); } - continue; + CONF_CONTINUE; } CONF_HANDLE_BOOL(opt_retain, "retain") if (strncmp("dss", k, klen) == 0) { @@ -1075,7 +1199,7 @@ malloc_conf_init(void) { if (strncmp(dss_prec_names[i], v, vlen) == 0) { if (extent_dss_prec_set(i)) { - malloc_conf_error( + CONF_ERROR( "Error setting dss", k, klen, v, vlen); } else { @@ -1087,13 +1211,36 @@ malloc_conf_init(void) { } } if (!match) { - malloc_conf_error("Invalid conf value", + CONF_ERROR("Invalid conf value", k, klen, v, vlen); } - continue; + CONF_CONTINUE; } CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, - UINT_MAX, yes, no, false) + UINT_MAX, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, + false) + if (CONF_MATCH("bin_shards")) { + const char *bin_shards_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t size_start; + size_t size_end; + size_t nshards; + bool err = malloc_conf_multi_sizes_next( + &bin_shards_segment_cur, &vlen_left, + &size_start, &size_end, &nshards); + if (err || bin_update_shard_size( + bin_shard_sizes, size_start, + size_end, nshards)) { + CONF_ERROR( + "Invalid settings for " + "bin_shards", k, klen, v, + vlen); + break; + } + } while (vlen_left > 0); + CONF_CONTINUE; + } CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : @@ -1105,7 +1252,7 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_stats_print, "stats_print") if (CONF_MATCH("stats_print_opts")) { init_opt_stats_print_opts(v, vlen); - continue; + CONF_CONTINUE; } if (config_fill) { if (CONF_MATCH("junk")) { @@ -1126,11 +1273,11 @@ malloc_conf_init(void) { opt_junk_alloc = false; opt_junk_free = true; } else { - malloc_conf_error( - "Invalid conf value", k, - klen, v, vlen); + CONF_ERROR( + "Invalid conf value", + k, klen, v, vlen); } - continue; + CONF_CONTINUE; } CONF_HANDLE_BOOL(opt_zero, "zero") } @@ -1141,11 +1288,25 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") } CONF_HANDLE_BOOL(opt_tcache, "tcache") - CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, - "lg_extent_max_active_fit", 0, - (sizeof(size_t) << 3), yes, yes, false) CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", -1, (sizeof(size_t) << 3) - 1) + + /* + * The runtime option of oversize_threshold remains + * undocumented. It may be tweaked in the next major + * release (6.0). The default value 8M is rather + * conservative / safe. Tuning it further down may + * improve fragmentation a bit more, but may also cause + * contention on the huge arena. + */ + CONF_HANDLE_SIZE_T(opt_oversize_threshold, + "oversize_threshold", 0, SC_LARGE_MAXCLASS, + CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false) + CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, + "lg_extent_max_active_fit", 0, + (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN, + CONF_CHECK_MAX, false) + if (strncmp("percpu_arena", k, klen) == 0) { bool match = false; for (int i = percpu_arena_mode_names_base; i < @@ -1153,7 +1314,7 @@ malloc_conf_init(void) { if (strncmp(percpu_arena_mode_names[i], v, vlen) == 0) { if (!have_percpu_arena) { - malloc_conf_error( + CONF_ERROR( "No getcpu support", k, klen, v, vlen); } @@ -1163,17 +1324,42 @@ malloc_conf_init(void) { } } if (!match) { - malloc_conf_error("Invalid conf value", + CONF_ERROR("Invalid conf value", k, klen, v, vlen); } - continue; + CONF_CONTINUE; } CONF_HANDLE_BOOL(opt_background_thread, "background_thread"); CONF_HANDLE_SIZE_T(opt_max_background_threads, "max_background_threads", 1, - opt_max_background_threads, yes, yes, + opt_max_background_threads, + CONF_CHECK_MIN, CONF_CHECK_MAX, true); + if (CONF_MATCH("slab_sizes")) { + bool err; + const char *slab_size_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t slab_start; + size_t slab_end; + size_t pgs; + err = malloc_conf_multi_sizes_next( + &slab_size_segment_cur, + &vlen_left, &slab_start, &slab_end, + &pgs); + if (!err) { + sc_data_update_slab_size( + sc_data, slab_start, + slab_end, (int)pgs); + } else { + CONF_ERROR("Invalid settings " + "for slab_sizes", + k, klen, v, vlen); + } + } while (!err && vlen_left > 0); + CONF_CONTINUE; + } if (config_prof) { CONF_HANDLE_BOOL(opt_prof, "prof") CONF_HANDLE_CHAR_P(opt_prof_prefix, @@ -1183,7 +1369,8 @@ malloc_conf_init(void) { "prof_thread_active_init") CONF_HANDLE_SIZE_T(opt_lg_prof_sample, "lg_prof_sample", 0, (sizeof(uint64_t) << 3) - - 1, no, yes, true) + - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, + true) CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, "lg_prof_interval", -1, @@ -1191,6 +1378,7 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") CONF_HANDLE_BOOL(opt_prof_final, "prof_final") CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") + CONF_HANDLE_BOOL(opt_prof_log, "prof_log") } if (config_log) { if (CONF_MATCH("log")) { @@ -1199,7 +1387,7 @@ malloc_conf_init(void) { vlen : sizeof(log_var_names) - 1); strncpy(log_var_names, v, cpylen); log_var_names[cpylen] = '\0'; - continue; + CONF_CONTINUE; } } if (CONF_MATCH("thp")) { @@ -1208,7 +1396,7 @@ malloc_conf_init(void) { if (strncmp(thp_mode_names[i],v, vlen) == 0) { if (!have_madvise_huge) { - malloc_conf_error( + CONF_ERROR( "No THP support", k, klen, v, vlen); } @@ -1218,25 +1406,28 @@ malloc_conf_init(void) { } } if (!match) { - malloc_conf_error("Invalid conf value", + CONF_ERROR("Invalid conf value", k, klen, v, vlen); } - continue; + CONF_CONTINUE; } - malloc_conf_error("Invalid conf pair", k, klen, v, - vlen); + CONF_ERROR("Invalid conf pair", k, klen, v, vlen); +#undef CONF_ERROR +#undef CONF_CONTINUE #undef CONF_MATCH #undef CONF_MATCH_VALUE #undef CONF_HANDLE_BOOL -#undef CONF_MIN_no -#undef CONF_MIN_yes -#undef CONF_MAX_no -#undef CONF_MAX_yes +#undef CONF_DONT_CHECK_MIN +#undef CONF_CHECK_MIN +#undef CONF_DONT_CHECK_MAX +#undef CONF_CHECK_MAX #undef CONF_HANDLE_T_U #undef CONF_HANDLE_UNSIGNED #undef CONF_HANDLE_SIZE_T #undef CONF_HANDLE_SSIZE_T #undef CONF_HANDLE_CHAR_P + /* Re-enable diagnostic "-Wtype-limits" */ + JEMALLOC_DIAGNOSTIC_POP } if (opt_abort_conf && had_conf_error) { malloc_abort_invalid_conf(); @@ -1245,6 +1436,19 @@ malloc_conf_init(void) { atomic_store_b(&log_init_done, true, ATOMIC_RELEASE); } +static void +malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { + const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL}; + char buf[PATH_MAX + 1]; + + /* The first call only set the confirm_conf option and opts_cache */ + malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf); + malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache, + NULL); +} + +#undef MALLOC_CONF_NSOURCES + static bool malloc_init_hard_needed(void) { if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == @@ -1275,10 +1479,33 @@ static bool malloc_init_hard_a0_locked() { malloc_initializer = INITIALIZER; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + sc_data_t sc_data = {0}; + JEMALLOC_DIAGNOSTIC_POP + + /* + * Ordering here is somewhat tricky; we need sc_boot() first, since that + * determines what the size classes will be, and then + * malloc_conf_init(), since any slab size tweaking will need to be done + * before sz_boot and bin_boot, which assume that the values they read + * out of sc_data_global are final. + */ + sc_boot(&sc_data); + unsigned bin_shard_sizes[SC_NBINS]; + bin_shard_sizes_boot(bin_shard_sizes); + /* + * prof_boot0 only initializes opt_prof_prefix. We need to do it before + * we parse malloc_conf options, in case malloc_conf parsing overwrites + * it. + */ if (config_prof) { prof_boot0(); } - malloc_conf_init(); + malloc_conf_init(&sc_data, bin_shard_sizes); + sz_boot(&sc_data); + bin_boot(&sc_data, bin_shard_sizes); + if (opt_stats_print) { /* Print statistics at exit. */ if (atexit(stats_print_atexit) != 0) { @@ -1303,7 +1530,7 @@ malloc_init_hard_a0_locked() { if (config_prof) { prof_boot1(); } - arena_boot(); + arena_boot(&sc_data); if (tcache_boot(TSDN_NULL)) { return true; } @@ -1311,11 +1538,13 @@ malloc_init_hard_a0_locked() { malloc_mutex_rank_exclusive)) { return true; } + hook_boot(); /* * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). */ narenas_auto = 1; + manual_arena_base = narenas_auto + 1; memset(arenas, 0, sizeof(arena_t *) * narenas_auto); /* * Initialize one arena here. The rest are lazily created in @@ -1463,6 +1692,10 @@ malloc_init_narenas(void) { narenas_auto); } narenas_total_set(narenas_auto); + if (arena_init_huge()) { + narenas_total_inc(); + } + manual_arena_base = narenas_total_get(); return false; } @@ -1560,11 +1793,7 @@ malloc_init_hard(void) { * sets isthreaded) needs to be called without holding any lock. */ background_thread_ctl_init(tsd_tsdn(tsd)); - - malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); - bool err = background_thread_create(tsd, 0); - malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); - if (err) { + if (background_thread_create(tsd, 0)) { return true; } } @@ -1587,8 +1816,12 @@ typedef struct static_opts_s static_opts_t; struct static_opts_s { /* Whether or not allocation size may overflow. */ bool may_overflow; - /* Whether or not allocations of size 0 should be treated as size 1. */ - bool bump_empty_alloc; + + /* + * Whether or not allocations (with alignment) of size 0 should be + * treated as size 1. + */ + bool bump_empty_aligned_alloc; /* * Whether to assert that allocations are not of size 0 (after any * bumping). @@ -1621,12 +1854,16 @@ struct static_opts_s { * initialization) options. */ bool slow; + /* + * Return size. + */ + bool usize; }; JEMALLOC_ALWAYS_INLINE void static_opts_init(static_opts_t *static_opts) { static_opts->may_overflow = false; - static_opts->bump_empty_alloc = false; + static_opts->bump_empty_aligned_alloc = false; static_opts->assert_nonempty_alloc = false; static_opts->null_out_result_on_error = false; static_opts->set_errno_on_error = false; @@ -1634,6 +1871,7 @@ static_opts_init(static_opts_t *static_opts) { static_opts->oom_string = ""; static_opts->invalid_alignment_string = ""; static_opts->slow = false; + static_opts->usize = false; } /* @@ -1648,6 +1886,7 @@ static_opts_init(static_opts_t *static_opts) { typedef struct dynamic_opts_s dynamic_opts_t; struct dynamic_opts_s { void **result; + size_t usize; size_t num_items; size_t item_size; size_t alignment; @@ -1659,6 +1898,7 @@ struct dynamic_opts_s { JEMALLOC_ALWAYS_INLINE void dynamic_opts_init(dynamic_opts_t *dynamic_opts) { dynamic_opts->result = NULL; + dynamic_opts->usize = 0; dynamic_opts->num_items = 0; dynamic_opts->item_size = 0; dynamic_opts->alignment = 0; @@ -1722,12 +1962,13 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, szind_t ind_large; size_t bumped_usize = usize; - if (usize <= SMALL_MAXCLASS) { - assert(((dopts->alignment == 0) ? sz_s2u(LARGE_MINCLASS) : - sz_sa2u(LARGE_MINCLASS, dopts->alignment)) - == LARGE_MINCLASS); - ind_large = sz_size2index(LARGE_MINCLASS); - bumped_usize = sz_s2u(LARGE_MINCLASS); + if (usize <= SC_SMALL_MAXCLASS) { + assert(((dopts->alignment == 0) ? + sz_s2u(SC_LARGE_MINCLASS) : + sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment)) + == SC_LARGE_MINCLASS); + ind_large = sz_size2index(SC_LARGE_MINCLASS); + bumped_usize = sz_s2u(SC_LARGE_MINCLASS); ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, bumped_usize, ind_large); if (unlikely(ret == NULL)) { @@ -1810,17 +2051,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { goto label_oom; } - /* Validate the user input. */ - if (sopts->bump_empty_alloc) { - if (unlikely(size == 0)) { - size = 1; - } - } - - if (sopts->assert_nonempty_alloc) { - assert (size != 0); - } - if (unlikely(dopts->alignment < sopts->min_alignment || (dopts->alignment & (dopts->alignment - 1)) != 0)) { goto label_invalid_alignment; @@ -1830,19 +2060,32 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { if (dopts->alignment == 0) { ind = sz_size2index(size); - if (unlikely(ind >= NSIZES)) { + if (unlikely(ind >= SC_NSIZES)) { goto label_oom; } - if (config_stats || (config_prof && opt_prof)) { + if (config_stats || (config_prof && opt_prof) || sopts->usize) { usize = sz_index2size(ind); - assert(usize > 0 && usize <= LARGE_MAXCLASS); + dopts->usize = usize; + assert(usize > 0 && usize + <= SC_LARGE_MAXCLASS); } } else { + if (sopts->bump_empty_aligned_alloc) { + if (unlikely(size == 0)) { + size = 1; + } + } usize = sz_sa2u(size, dopts->alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + dopts->usize = usize; + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { goto label_oom; } } + /* Validate the user input. */ + if (sopts->assert_nonempty_alloc) { + assert (size != 0); + } check_entry_exit_locking(tsd_tsdn(tsd)); @@ -1875,7 +2118,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { alloc_ctx_t alloc_ctx; if (likely((uintptr_t)tctx == (uintptr_t)1U)) { - alloc_ctx.slab = (usize <= SMALL_MAXCLASS); + alloc_ctx.slab = (usize + <= SC_SMALL_MAXCLASS); allocation = imalloc_no_sample( sopts, dopts, tsd, usize, usize, ind); } else if ((uintptr_t)tctx > (uintptr_t)1U) { @@ -1980,9 +2224,8 @@ label_invalid_alignment: return EINVAL; } -/* Returns the errno-style error code of the allocation. */ -JEMALLOC_ALWAYS_INLINE int -imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { +JEMALLOC_ALWAYS_INLINE bool +imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) { if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) { if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(sopts->oom_string); @@ -1992,6 +2235,16 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { set_errno(ENOMEM); *dopts->result = NULL; + return false; + } + + return true; +} + +/* Returns the errno-style error code of the allocation. */ +JEMALLOC_ALWAYS_INLINE int +imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { + if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { return ENOMEM; } @@ -2004,19 +2257,18 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { sopts->slow = false; return imalloc_body(sopts, dopts, tsd); } else { + if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { + return ENOMEM; + } + sopts->slow = true; return imalloc_body(sopts, dopts, tsd); } } -/******************************************************************************/ -/* - * Begin malloc(3)-compatible functions. - */ -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) -je_malloc(size_t size) { +JEMALLOC_NOINLINE +void * +malloc_default(size_t size) { void *ret; static_opts_t sopts; dynamic_opts_t dopts; @@ -2026,7 +2278,6 @@ je_malloc(size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = ": Error in malloc(): out of memory\n"; @@ -2036,12 +2287,107 @@ je_malloc(size_t size) { dopts.item_size = size; imalloc(&sopts, &dopts); + /* + * Note that this branch gets optimized away -- it immediately follows + * the check on tsd_fast that sets sopts.slow. + */ + if (sopts.slow) { + uintptr_t args[3] = {size}; + hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args); + } LOG("core.malloc.exit", "result: %p", ret); return ret; } +/******************************************************************************/ +/* + * Begin malloc(3)-compatible functions. + */ + +/* + * malloc() fastpath. + * + * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit + * tcache. If either of these is false, we tail-call to the slowpath, + * malloc_default(). Tail-calling is used to avoid any caller-saved + * registers. + * + * fastpath supports ticker and profiling, both of which will also + * tail-call to the slowpath if they fire. + */ +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN +void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) +je_malloc(size_t size) { + LOG("core.malloc.entry", "size: %zu", size); + + if (tsd_get_allocates() && unlikely(!malloc_initialized())) { + return malloc_default(size); + } + + tsd_t *tsd = tsd_get(false); + if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) { + return malloc_default(size); + } + + tcache_t *tcache = tsd_tcachep_get(tsd); + + if (unlikely(ticker_trytick(&tcache->gc_ticker))) { + return malloc_default(size); + } + + szind_t ind = sz_size2index_lookup(size); + size_t usize; + if (config_stats || config_prof) { + usize = sz_index2size(ind); + } + /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */ + assert(ind < SC_NBINS); + assert(size <= SC_SMALL_MAXCLASS); + + if (config_prof) { + int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); + bytes_until_sample -= usize; + tsd_bytes_until_sample_set(tsd, bytes_until_sample); + + if (unlikely(bytes_until_sample < 0)) { + /* + * Avoid a prof_active check on the fastpath. + * If prof_active is false, set bytes_until_sample to + * a large value. If prof_active is set to true, + * bytes_until_sample will be reset. + */ + if (!prof_active) { + tsd_bytes_until_sample_set(tsd, SSIZE_MAX); + } + return malloc_default(size); + } + } + + cache_bin_t *bin = tcache_small_bin_get(tcache, ind); + bool tcache_success; + void* ret = cache_bin_alloc_easy(bin, &tcache_success); + + if (tcache_success) { + if (config_stats) { + *tsd_thread_allocatedp_get(tsd) += usize; + bin->tstats.nrequests++; + } + if (config_prof) { + tcache->prof_accumbytes += usize; + } + + LOG("core.malloc.exit", "result: %p", ret); + + /* Fastpath success */ + return ret; + } + + return malloc_default(size); +} + JEMALLOC_EXPORT int JEMALLOC_NOTHROW JEMALLOC_ATTR(nonnull(1)) je_posix_memalign(void **memptr, size_t alignment, size_t size) { @@ -2055,7 +2401,7 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; + sopts.bump_empty_aligned_alloc = true; sopts.min_alignment = sizeof(void *); sopts.oom_string = ": Error allocating aligned memory: out of memory\n"; @@ -2068,6 +2414,12 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { dopts.alignment = alignment; ret = imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment, + (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_posix_memalign, *memptr, + (uintptr_t)ret, args); + } LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret, *memptr); @@ -2090,7 +2442,7 @@ je_aligned_alloc(size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; + sopts.bump_empty_aligned_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.min_alignment = 1; @@ -2105,6 +2457,11 @@ je_aligned_alloc(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_aligned_alloc, ret, + (uintptr_t)ret, args); + } LOG("core.aligned_alloc.exit", "result: %p", ret); @@ -2125,7 +2482,6 @@ je_calloc(size_t num, size_t size) { dynamic_opts_init(&dopts); sopts.may_overflow = true; - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = ": Error in calloc(): out of memory\n"; @@ -2136,6 +2492,10 @@ je_calloc(size_t num, size_t size) { dopts.zero = true; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args); + } LOG("core.calloc.exit", "result: %p", ret); @@ -2144,20 +2504,22 @@ je_calloc(size_t num, size_t size) { static void * irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - prof_tctx_t *tctx) { + prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SMALL_MAXCLASS) { - p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); + if (usize <= SC_SMALL_MAXCLASS) { + p = iralloc(tsd, old_ptr, old_usize, + SC_LARGE_MINCLASS, 0, false, hook_args); if (p == NULL) { return NULL; } arena_prof_promote(tsd_tsdn(tsd), p, usize); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, + hook_args); } return p; @@ -2165,7 +2527,7 @@ irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, JEMALLOC_ALWAYS_INLINE void * irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - alloc_ctx_t *alloc_ctx) { + alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2174,9 +2536,11 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); tctx = prof_alloc_prep(tsd, usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); + p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx, + hook_args); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, + hook_args); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); @@ -2205,7 +2569,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); size_t usize; if (config_prof && opt_prof) { @@ -2286,11 +2650,12 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ALLOC_SIZE(2) -je_realloc(void *ptr, size_t size) { +je_realloc(void *ptr, size_t arg_size) { void *ret; tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); size_t usize JEMALLOC_CC_SILENCE_INIT(0); size_t old_usize = 0; + size_t size = arg_size; LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); @@ -2305,6 +2670,10 @@ je_realloc(void *ptr, size_t size) { } else { tcache = NULL; } + + uintptr_t args[3] = {(uintptr_t)ptr, size}; + hook_invoke_dalloc(hook_dalloc_realloc, ptr, args); + ifree(tsd, ptr, tcache, true); LOG("core.realloc.exit", "result: %p", NULL); @@ -2319,29 +2688,58 @@ je_realloc(void *ptr, size_t size) { check_entry_exit_locking(tsd_tsdn(tsd)); + + hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr, + (uintptr_t)arg_size, 0, 0}}; + alloc_ctx_t alloc_ctx; rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); if (config_prof && opt_prof) { usize = sz_s2u(size); - ret = unlikely(usize == 0 || usize > LARGE_MAXCLASS) ? - NULL : irealloc_prof(tsd, ptr, old_usize, usize, - &alloc_ctx); + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { + ret = NULL; + } else { + ret = irealloc_prof(tsd, ptr, old_usize, usize, + &alloc_ctx, &hook_args); + } } else { if (config_stats) { usize = sz_s2u(size); } - ret = iralloc(tsd, ptr, old_usize, size, 0, false); + ret = iralloc(tsd, ptr, old_usize, size, 0, false, + &hook_args); } tsdn = tsd_tsdn(tsd); } else { /* realloc(NULL, size) is equivalent to malloc(size). */ - void *ret = je_malloc(size); - LOG("core.realloc.exit", "result: %p", ret); + static_opts_t sopts; + dynamic_opts_t dopts; + + static_opts_init(&sopts); + dynamic_opts_init(&dopts); + + sopts.null_out_result_on_error = true; + sopts.set_errno_on_error = true; + sopts.oom_string = + ": Error in realloc(): out of memory\n"; + + dopts.result = &ret; + dopts.num_items = 1; + dopts.item_size = size; + + imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)ptr, arg_size}; + hook_invoke_alloc(hook_alloc_realloc, ret, + (uintptr_t)ret, args); + } + return ret; } @@ -2368,10 +2766,9 @@ je_realloc(void *ptr, size_t size) { return ret; } -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_free(void *ptr) { - LOG("core.free.entry", "ptr: %p", ptr); - +JEMALLOC_NOINLINE +void +free_default(void *ptr) { UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) { /* @@ -2397,10 +2794,79 @@ je_free(void *ptr) { } else { tcache = NULL; } + uintptr_t args_raw[3] = {(uintptr_t)ptr}; + hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); } +} + +JEMALLOC_ALWAYS_INLINE +bool free_fastpath(void *ptr, size_t size, bool size_hint) { + tsd_t *tsd = tsd_get(false); + if (unlikely(!tsd || !tsd_fast(tsd))) { + return false; + } + + tcache_t *tcache = tsd_tcachep_get(tsd); + + alloc_ctx_t alloc_ctx; + /* + * If !config_cache_oblivious, we can check PAGE alignment to + * detect sampled objects. Otherwise addresses are + * randomized, and we have to look it up in the rtree anyway. + * See also isfree(). + */ + if (!size_hint || config_cache_oblivious) { + rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); + bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree, + rtree_ctx, (uintptr_t)ptr, + &alloc_ctx.szind, &alloc_ctx.slab); + + /* Note: profiled objects will have alloc_ctx.slab set */ + if (!res || !alloc_ctx.slab) { + return false; + } + assert(alloc_ctx.szind != SC_NSIZES); + } else { + /* + * Check for both sizes that are too large, and for sampled objects. + * Sampled objects are always page-aligned. The sampled object check + * will also check for null ptr. + */ + if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) { + return false; + } + alloc_ctx.szind = sz_size2index_lookup(size); + } + + if (unlikely(ticker_trytick(&tcache->gc_ticker))) { + return false; + } + + cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind); + cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind]; + if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) { + return false; + } + + if (config_stats) { + size_t usize = sz_index2size(alloc_ctx.szind); + *tsd_thread_deallocatedp_get(tsd) += usize; + } + + return true; +} + +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_free(void *ptr) { + LOG("core.free.entry", "ptr: %p", ptr); + + if (!free_fastpath(ptr, 0, false)) { + free_default(ptr); + } + LOG("core.free.exit", ""); } @@ -2427,7 +2893,6 @@ je_memalign(size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.min_alignment = 1; sopts.oom_string = ": Error allocating aligned memory: out of memory\n"; @@ -2441,6 +2906,11 @@ je_memalign(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {alignment, size}; + hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret, + args); + } LOG("core.memalign.exit", "result: %p", ret); return ret; @@ -2462,7 +2932,6 @@ je_valloc(size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.min_alignment = PAGE; sopts.oom_string = @@ -2476,6 +2945,10 @@ je_valloc(size_t size) { dopts.alignment = PAGE; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {size}; + hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args); + } LOG("core.valloc.exit", "result: %p\n", ret); return ret; @@ -2543,6 +3016,82 @@ int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign); * Begin non-standard functions. */ +#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API + +#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y +#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ + JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) + +typedef struct { + void *ptr; + size_t size; +} smallocx_return_t; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN +smallocx_return_t JEMALLOC_NOTHROW +/* + * The attribute JEMALLOC_ATTR(malloc) cannot be used due to: + * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488 + */ +JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) + (size_t size, int flags) { + /* + * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be + * used here because it makes writing beyond the `size` + * of the `ptr` undefined behavior, but the objective + * of this function is to allow writing beyond `size` + * up to `smallocx_return_t::size`. + */ + smallocx_return_t ret; + static_opts_t sopts; + dynamic_opts_t dopts; + + LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags); + + static_opts_init(&sopts); + dynamic_opts_init(&dopts); + + sopts.assert_nonempty_alloc = true; + sopts.null_out_result_on_error = true; + sopts.oom_string = ": Error in mallocx(): out of memory\n"; + sopts.usize = true; + + dopts.result = &ret.ptr; + dopts.num_items = 1; + dopts.item_size = size; + if (unlikely(flags != 0)) { + if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) { + dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); + } + + dopts.zero = MALLOCX_ZERO_GET(flags); + + if ((flags & MALLOCX_TCACHE_MASK) != 0) { + if ((flags & MALLOCX_TCACHE_MASK) + == MALLOCX_TCACHE_NONE) { + dopts.tcache_ind = TCACHE_IND_NONE; + } else { + dopts.tcache_ind = MALLOCX_TCACHE_GET(flags); + } + } else { + dopts.tcache_ind = TCACHE_IND_AUTOMATIC; + } + + if ((flags & MALLOCX_ARENA_MASK) != 0) + dopts.arena_ind = MALLOCX_ARENA_GET(flags); + } + + imalloc(&sopts, &dopts); + assert(dopts.usize == je_nallocx(size, flags)); + ret.size = dopts.usize; + + LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size); + return ret; +} +#undef JEMALLOC_SMALLOCX_CONCAT_HELPER +#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2 +#endif + JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) @@ -2586,6 +3135,11 @@ je_mallocx(size_t size, int flags) { } imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {size, flags}; + hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret, + args); + } LOG("core.mallocx.exit", "result: %p", ret); return ret; @@ -2594,22 +3148,23 @@ je_mallocx(size_t size, int flags) { static void * irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, - prof_tctx_t *tctx) { + prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SMALL_MAXCLASS) { - p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS, - alignment, zero, tcache, arena); + if (usize <= SC_SMALL_MAXCLASS) { + p = iralloct(tsdn, old_ptr, old_usize, + SC_LARGE_MINCLASS, alignment, zero, tcache, + arena, hook_args); if (p == NULL) { return NULL; } arena_prof_promote(tsdn, p, usize); } else { p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, - tcache, arena); + tcache, arena, hook_args); } return p; @@ -2618,7 +3173,7 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, JEMALLOC_ALWAYS_INLINE void * irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache, - arena_t *arena, alloc_ctx_t *alloc_ctx) { + arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2628,10 +3183,10 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, tctx = prof_alloc_prep(tsd, *usize, prof_active, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, - *usize, alignment, zero, tcache, arena, tctx); + *usize, alignment, zero, tcache, arena, tctx, hook_args); } else { p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, - zero, tcache, arena); + zero, tcache, arena, hook_args); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, false); @@ -2702,23 +3257,27 @@ je_rallocx(void *ptr, size_t size, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); + + hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags, + 0}}; if (config_prof && opt_prof) { usize = (alignment == 0) ? sz_s2u(size) : sz_sa2u(size, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { goto label_oom; } p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, - zero, tcache, arena, &alloc_ctx); + zero, tcache, arena, &alloc_ctx, &hook_args); if (unlikely(p == NULL)) { goto label_oom; } } else { p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, - zero, tcache, arena); + zero, tcache, arena, &hook_args); if (unlikely(p == NULL)) { goto label_oom; } @@ -2752,14 +3311,14 @@ label_oom: JEMALLOC_ALWAYS_INLINE size_t ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero) { - size_t usize; + size_t newsize; - if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) { + if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero, + &newsize)) { return old_usize; } - usize = isalloc(tsdn, ptr); - return usize; + return newsize; } static size_t @@ -2793,17 +3352,19 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, */ if (alignment == 0) { usize_max = sz_s2u(size+extra); - assert(usize_max > 0 && usize_max <= LARGE_MAXCLASS); + assert(usize_max > 0 + && usize_max <= SC_LARGE_MAXCLASS); } else { usize_max = sz_sa2u(size+extra, alignment); - if (unlikely(usize_max == 0 || usize_max > LARGE_MAXCLASS)) { + if (unlikely(usize_max == 0 + || usize_max > SC_LARGE_MAXCLASS)) { /* * usize_max is out of range, and chances are that * allocation will fail, but use the maximum possible * value and carry on with prof_alloc_prep(), just in * case allocation succeeds. */ - usize_max = LARGE_MAXCLASS; + usize_max = SC_LARGE_MAXCLASS; } } tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); @@ -2846,24 +3407,24 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); /* * The API explicitly absolves itself of protecting against (size + * extra) numerical overflow, but we may need to clamp extra to avoid - * exceeding LARGE_MAXCLASS. + * exceeding SC_LARGE_MAXCLASS. * * Ordinarily, size limit checking is handled deeper down, but here we * have to check as part of (size + extra) clamping, since we need the * clamped value in the above helper functions. */ - if (unlikely(size > LARGE_MAXCLASS)) { + if (unlikely(size > SC_LARGE_MAXCLASS)) { usize = old_usize; goto label_not_resized; } - if (unlikely(LARGE_MAXCLASS - size < extra)) { - extra = LARGE_MAXCLASS - size; + if (unlikely(SC_LARGE_MAXCLASS - size < extra)) { + extra = SC_LARGE_MAXCLASS - size; } if (config_prof && opt_prof) { @@ -2882,6 +3443,12 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { *tsd_thread_deallocatedp_get(tsd) += old_usize; } label_not_resized: + if (unlikely(!tsd_fast(tsd))) { + uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; + hook_invoke_expand(hook_expand_xallocx, ptr, old_usize, + usize, (uintptr_t)usize, args); + } + UTRACE(ptr, size, ptr); check_entry_exit_locking(tsd_tsdn(tsd)); @@ -2891,7 +3458,7 @@ label_not_resized: JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW JEMALLOC_ATTR(pure) -je_sallocx(const void *ptr, UNUSED int flags) { +je_sallocx(const void *ptr, int flags) { size_t usize; tsdn_t *tsdn; @@ -2954,6 +3521,8 @@ je_dallocx(void *ptr, int flags) { tsd_assert_fast(tsd); ifree(tsd, ptr, tcache, false); } else { + uintptr_t args_raw[3] = {(uintptr_t)ptr, flags}; + hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); @@ -2975,14 +3544,11 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { return usize; } -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_sdallocx(void *ptr, size_t size, int flags) { +JEMALLOC_NOINLINE void +sdallocx_default(void *ptr, size_t size, int flags) { assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); - LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, - size, flags); - tsd_t *tsd = tsd_fetch(); bool fast = tsd_fast(tsd); size_t usize = inallocx(tsd_tsdn(tsd), size, flags); @@ -3016,10 +3582,35 @@ je_sdallocx(void *ptr, size_t size, int flags) { tsd_assert_fast(tsd); isfree(tsd, ptr, usize, tcache, false); } else { + uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags}; + hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw); isfree(tsd, ptr, usize, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); +} + +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_sdallocx(void *ptr, size_t size, int flags) { + LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, + size, flags); + + if (flags !=0 || !free_fastpath(ptr, size, true)) { + sdallocx_default(ptr, size, flags); + } + + LOG("core.sdallocx.exit", ""); +} + +void JEMALLOC_NOTHROW +je_sdallocx_noflags(void *ptr, size_t size) { + LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr, + size); + + if (!free_fastpath(ptr, size, true)) { + sdallocx_default(ptr, size, 0); + } + LOG("core.sdallocx.exit", ""); } @@ -3040,7 +3631,7 @@ je_nallocx(size_t size, int flags) { check_entry_exit_locking(tsdn); usize = inallocx(tsdn, size, flags); - if (unlikely(usize > LARGE_MAXCLASS)) { + if (unlikely(usize > SC_LARGE_MAXCLASS)) { LOG("core.nallocx.exit", "result: %zu", ZU(0)); return 0; } @@ -3256,6 +3847,7 @@ _malloc_prefork(void) } } prof_prefork1(tsd_tsdn(tsd)); + tsd_prefork(tsd); } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -3278,6 +3870,8 @@ _malloc_postfork(void) tsd = tsd_fetch(); + tsd_postfork_parent(tsd); + witness_postfork_parent(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { @@ -3305,6 +3899,8 @@ jemalloc_postfork_child(void) { tsd = tsd_fetch(); + tsd_postfork_child(tsd); + witness_postfork_child(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { diff --git a/deps/jemalloc/src/jemalloc_cpp.cpp b/deps/jemalloc/src/jemalloc_cpp.cpp index f0ceddae3..da0441a7c 100644 --- a/deps/jemalloc/src/jemalloc_cpp.cpp +++ b/deps/jemalloc/src/jemalloc_cpp.cpp @@ -128,14 +128,14 @@ operator delete(void *ptr, std::size_t size) noexcept { if (unlikely(ptr == nullptr)) { return; } - je_sdallocx(ptr, size, /*flags=*/0); + je_sdallocx_noflags(ptr, size); } void operator delete[](void *ptr, std::size_t size) noexcept { if (unlikely(ptr == nullptr)) { return; } - je_sdallocx(ptr, size, /*flags=*/0); + je_sdallocx_noflags(ptr, size); } #endif // __cpp_sized_deallocation diff --git a/deps/jemalloc/src/large.c b/deps/jemalloc/src/large.c index 27a2c6798..8e7a781d3 100644 --- a/deps/jemalloc/src/large.c +++ b/deps/jemalloc/src/large.c @@ -28,7 +28,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, assert(!tsdn_null(tsdn) || arena != NULL); ausize = sz_sa2u(usize, alignment); - if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS)) { + if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { return NULL; } @@ -42,7 +42,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, */ is_zeroed = zero; if (likely(!tsdn_null(tsdn))) { - arena = arena_choose(tsdn_tsd(tsdn), arena); + arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); } if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn, arena, usize, alignment, &is_zeroed)) == NULL) { @@ -109,7 +109,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { if (diff != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, &extent_hooks, extent, usize + sz_large_pad, - sz_size2index(usize), false, diff, NSIZES, false); + sz_size2index(usize), false, diff, SC_NSIZES, false); if (trail == NULL) { return true; } @@ -154,17 +154,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool new_mapping; if ((trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, extent_past_get(extent), trailsize, 0, - CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL + CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL || (trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, extent_past_get(extent), trailsize, 0, - CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL) { + CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { if (config_stats) { new_mapping = false; } } else { if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, extent_past_get(extent), trailsize, 0, CACHELINE, false, - NSIZES, &is_zeroed_trail, &commit)) == NULL) { + SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) { return true; } if (config_stats) { @@ -221,9 +221,10 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t oldusize = extent_usize_get(extent); /* The following should have been caught by callers. */ - assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS); + assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS + && usize_max >= SC_LARGE_MINCLASS); if (usize_max > oldusize) { /* Attempt to expand the allocation in-place. */ @@ -270,17 +271,23 @@ large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, } void * -large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, - size_t alignment, bool zero, tcache_t *tcache) { - size_t oldusize = extent_usize_get(extent); +large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, + size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args) { + extent_t *extent = iealloc(tsdn, ptr); + size_t oldusize = extent_usize_get(extent); /* The following should have been caught by callers. */ - assert(usize > 0 && usize <= LARGE_MAXCLASS); + assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS + && usize >= SC_LARGE_MINCLASS); /* Try to avoid moving the allocation. */ if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) { + hook_invoke_expand(hook_args->is_realloc + ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, + usize, (uintptr_t)ptr, hook_args->args); return extent_addr_get(extent); } @@ -295,6 +302,12 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, return NULL; } + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + size_t copysize = (usize < oldusize) ? usize : oldusize; memcpy(ret, extent_addr_get(extent), copysize); isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true); @@ -318,8 +331,9 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, large_dalloc_maybe_junk(extent_addr_get(extent), extent_usize_get(extent)); } else { - malloc_mutex_assert_owner(tsdn, &arena->large_mtx); + /* Only hold the large_mtx if necessary. */ if (!arena_is_auto(arena)) { + malloc_mutex_assert_owner(tsdn, &arena->large_mtx); extent_list_remove(&arena->large, extent); } } @@ -369,3 +383,13 @@ void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) { large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U); } + +nstime_t +large_prof_alloc_time_get(const extent_t *extent) { + return extent_prof_alloc_time_get(extent); +} + +void +large_prof_alloc_time_set(extent_t *extent, nstime_t t) { + extent_prof_alloc_time_set(extent, t); +} diff --git a/deps/jemalloc/src/malloc_io.c b/deps/jemalloc/src/malloc_io.c index 7bdc13f95..d7cb0f528 100644 --- a/deps/jemalloc/src/malloc_io.c +++ b/deps/jemalloc/src/malloc_io.c @@ -362,7 +362,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { } \ } while (0) #define GET_ARG_NUMERIC(val, len) do { \ - switch (len) { \ + switch ((unsigned char)len) { \ case '?': \ val = va_arg(ap, int); \ break; \ @@ -632,7 +632,6 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, */ write_cb = (je_malloc_message != NULL) ? je_malloc_message : wrtmessage; - cbopaque = NULL; } malloc_vsnprintf(buf, sizeof(buf), format, ap); diff --git a/deps/jemalloc/src/mutex.c b/deps/jemalloc/src/mutex.c index 30222b3e5..3f920f5b1 100644 --- a/deps/jemalloc/src/mutex.c +++ b/deps/jemalloc/src/mutex.c @@ -46,7 +46,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { mutex_prof_data_t *data = &mutex->prof_data; - UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER; + nstime_t before = NSTIME_ZERO_INITIALIZER; if (ncpus == 1) { goto label_spin_done; @@ -55,7 +55,8 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) { int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN; do { spin_cpu_spinwait(); - if (!malloc_mutex_trylock_final(mutex)) { + if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED) + && !malloc_mutex_trylock_final(mutex)) { data->n_spin_acquired++; return; } @@ -144,9 +145,7 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, } # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - mutex->lock = OS_UNFAIR_LOCK_INIT; -#elif (defined(JEMALLOC_OSSPIN)) - mutex->lock = 0; + mutex->lock = OS_UNFAIR_LOCK_INIT; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) if (postpone_init) { mutex->postponed_next = postponed_mutexes; diff --git a/deps/jemalloc/src/pages.c b/deps/jemalloc/src/pages.c index 26002692d..13de27a00 100644 --- a/deps/jemalloc/src/pages.c +++ b/deps/jemalloc/src/pages.c @@ -180,6 +180,35 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { assert(alignment >= PAGE); assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr); +#if defined(__FreeBSD__) && defined(MAP_EXCL) + /* + * FreeBSD has mechanisms both to mmap at specific address without + * touching existing mappings, and to mmap with specific alignment. + */ + { + if (os_overcommits) { + *commit = true; + } + + int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; + int flags = mmap_flags; + + if (addr != NULL) { + flags |= MAP_FIXED | MAP_EXCL; + } else { + unsigned alignment_bits = ffs_zu(alignment); + assert(alignment_bits > 1); + flags |= MAP_ALIGNED(alignment_bits - 1); + } + + void *ret = mmap(addr, size, prot, flags, -1, 0); + if (ret == MAP_FAILED) { + ret = NULL; + } + + return ret; + } +#endif /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work @@ -261,7 +290,7 @@ pages_decommit(void *addr, size_t size) { bool pages_purge_lazy(void *addr, size_t size) { - assert(PAGE_ADDR2BASE(addr) == addr); + assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); assert(PAGE_CEILING(size) == size); if (!pages_can_purge_lazy) { @@ -391,6 +420,10 @@ os_page_detect(void) { GetSystemInfo(&si); return si.dwPageSize; #elif defined(__FreeBSD__) + /* + * This returns the value obtained from + * the auxv vector, avoiding a syscall. + */ return getpagesize(); #else long result = sysconf(_SC_PAGESIZE); @@ -544,6 +577,10 @@ init_thp_state(void) { close(fd); #endif + if (nread < 0) { + goto label_error; + } + if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) { init_system_thp_mode = thp_mode_default; } else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) { @@ -588,6 +625,11 @@ pages_boot(void) { init_thp_state(); +#ifdef __FreeBSD__ + /* + * FreeBSD doesn't need the check; madvise(2) is known to work. + */ +#else /* Detect lazy purge runtime support. */ if (pages_can_purge_lazy) { bool committed = false; @@ -601,6 +643,7 @@ pages_boot(void) { } os_pages_unmap(madv_free_page, PAGE); } +#endif return false; } diff --git a/deps/jemalloc/src/prof.c b/deps/jemalloc/src/prof.c index 13df641a0..13334cb4c 100644 --- a/deps/jemalloc/src/prof.c +++ b/deps/jemalloc/src/prof.c @@ -7,6 +7,7 @@ #include "jemalloc/internal/hash.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/emitter.h" /******************************************************************************/ @@ -23,7 +24,7 @@ */ #undef _Unwind_Backtrace #include -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) #endif /******************************************************************************/ @@ -38,6 +39,7 @@ bool opt_prof_gdump = false; bool opt_prof_final = false; bool opt_prof_leak = false; bool opt_prof_accum = false; +bool opt_prof_log = false; char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -70,6 +72,100 @@ uint64_t prof_interval = 0; size_t lg_prof_sample; +typedef enum prof_logging_state_e prof_logging_state_t; +enum prof_logging_state_e { + prof_logging_state_stopped, + prof_logging_state_started, + prof_logging_state_dumping +}; + +/* + * - stopped: log_start never called, or previous log_stop has completed. + * - started: log_start called, log_stop not called yet. Allocations are logged. + * - dumping: log_stop called but not finished; samples are not logged anymore. + */ +prof_logging_state_t prof_logging_state = prof_logging_state_stopped; + +#ifdef JEMALLOC_JET +static bool prof_log_dummy = false; +#endif + +/* Incremented for every log file that is output. */ +static uint64_t log_seq = 0; +static char log_filename[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PATH_MAX + +#endif + 1]; + +/* Timestamp for most recent call to log_start(). */ +static nstime_t log_start_timestamp = NSTIME_ZERO_INITIALIZER; + +/* Increment these when adding to the log_bt and log_thr linked lists. */ +static size_t log_bt_index = 0; +static size_t log_thr_index = 0; + +/* Linked list node definitions. These are only used in prof.c. */ +typedef struct prof_bt_node_s prof_bt_node_t; + +struct prof_bt_node_s { + prof_bt_node_t *next; + size_t index; + prof_bt_t bt; + /* Variable size backtrace vector pointed to by bt. */ + void *vec[1]; +}; + +typedef struct prof_thr_node_s prof_thr_node_t; + +struct prof_thr_node_s { + prof_thr_node_t *next; + size_t index; + uint64_t thr_uid; + /* Variable size based on thr_name_sz. */ + char name[1]; +}; + +typedef struct prof_alloc_node_s prof_alloc_node_t; + +/* This is output when logging sampled allocations. */ +struct prof_alloc_node_s { + prof_alloc_node_t *next; + /* Indices into an array of thread data. */ + size_t alloc_thr_ind; + size_t free_thr_ind; + + /* Indices into an array of backtraces. */ + size_t alloc_bt_ind; + size_t free_bt_ind; + + uint64_t alloc_time_ns; + uint64_t free_time_ns; + + size_t usize; +}; + +/* + * Created on the first call to prof_log_start and deleted on prof_log_stop. + * These are the backtraces and threads that have already been logged by an + * allocation. + */ +static bool log_tables_initialized = false; +static ckh_t log_bt_node_set; +static ckh_t log_thr_node_set; + +/* Store linked lists for logged data. */ +static prof_bt_node_t *log_bt_first = NULL; +static prof_bt_node_t *log_bt_last = NULL; +static prof_thr_node_t *log_thr_first = NULL; +static prof_thr_node_t *log_thr_last = NULL; +static prof_alloc_node_t *log_alloc_first = NULL; +static prof_alloc_node_t *log_alloc_last = NULL; + +/* Protects the prof_logging_state and any log_{...} variable. */ +static malloc_mutex_t log_mtx; + /* * Table of mutexes that are shared among gctx's. These are leaf locks, so * there is no problem with using them for more than one gctx at the same time. @@ -145,6 +241,12 @@ static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached); static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); +/* Hashtable functions for log_bt_node_set and log_thr_node_set. */ +static void prof_thr_node_hash(const void *key, size_t r_hash[2]); +static bool prof_thr_node_keycomp(const void *k1, const void *k2); +static void prof_bt_node_hash(const void *key, size_t r_hash[2]); +static bool prof_bt_node_keycomp(const void *k1, const void *k2); + /******************************************************************************/ /* Red-black trees. */ @@ -242,6 +344,12 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) { prof_tctx_set(tsdn, ptr, usize, NULL, tctx); + /* Get the current time and set this in the extent_t. We'll read this + * when free() is called. */ + nstime_t t = NSTIME_ZERO_INITIALIZER; + nstime_update(&t); + prof_alloc_time_set(tsdn, ptr, NULL, t); + malloc_mutex_lock(tsdn, tctx->tdata->lock); tctx->cnts.curobjs++; tctx->cnts.curbytes += usize; @@ -253,14 +361,174 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, malloc_mutex_unlock(tsdn, tctx->tdata->lock); } +static size_t +prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) { + assert(prof_logging_state == prof_logging_state_started); + malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); + + prof_bt_node_t dummy_node; + dummy_node.bt = *bt; + prof_bt_node_t *node; + + /* See if this backtrace is already cached in the table. */ + if (ckh_search(&log_bt_node_set, (void *)(&dummy_node), + (void **)(&node), NULL)) { + size_t sz = offsetof(prof_bt_node_t, vec) + + (bt->len * sizeof(void *)); + prof_bt_node_t *new_node = (prof_bt_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, + true, arena_get(TSDN_NULL, 0, true), true); + if (log_bt_first == NULL) { + log_bt_first = new_node; + log_bt_last = new_node; + } else { + log_bt_last->next = new_node; + log_bt_last = new_node; + } + + new_node->next = NULL; + new_node->index = log_bt_index; + /* + * Copy the backtrace: bt is inside a tdata or gctx, which + * might die before prof_log_stop is called. + */ + new_node->bt.len = bt->len; + memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *)); + new_node->bt.vec = new_node->vec; + + log_bt_index++; + ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL); + return new_node->index; + } else { + return node->index; + } +} +static size_t +prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) { + assert(prof_logging_state == prof_logging_state_started); + malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); + + prof_thr_node_t dummy_node; + dummy_node.thr_uid = thr_uid; + prof_thr_node_t *node; + + /* See if this thread is already cached in the table. */ + if (ckh_search(&log_thr_node_set, (void *)(&dummy_node), + (void **)(&node), NULL)) { + size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1; + prof_thr_node_t *new_node = (prof_thr_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, + true, arena_get(TSDN_NULL, 0, true), true); + if (log_thr_first == NULL) { + log_thr_first = new_node; + log_thr_last = new_node; + } else { + log_thr_last->next = new_node; + log_thr_last = new_node; + } + + new_node->next = NULL; + new_node->index = log_thr_index; + new_node->thr_uid = thr_uid; + strcpy(new_node->name, name); + + log_thr_index++; + ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL); + return new_node->index; + } else { + return node->index; + } +} + +static void +prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) { + malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); + + prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false); + if (cons_tdata == NULL) { + /* + * We decide not to log these allocations. cons_tdata will be + * NULL only when the current thread is in a weird state (e.g. + * it's being destroyed). + */ + return; + } + + malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx); + + if (prof_logging_state != prof_logging_state_started) { + goto label_done; + } + + if (!log_tables_initialized) { + bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, + prof_bt_node_hash, prof_bt_node_keycomp); + bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, + prof_thr_node_hash, prof_thr_node_keycomp); + if (err1 || err2) { + goto label_done; + } + log_tables_initialized = true; + } + + nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr, + (alloc_ctx_t *)NULL); + nstime_t free_time = NSTIME_ZERO_INITIALIZER; + nstime_update(&free_time); + + size_t sz = sizeof(prof_alloc_node_t); + prof_alloc_node_t *new_node = (prof_alloc_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, + arena_get(TSDN_NULL, 0, true), true); + + const char *prod_thr_name = (tctx->tdata->thread_name == NULL)? + "" : tctx->tdata->thread_name; + const char *cons_thr_name = prof_thread_name_get(tsd); + + prof_bt_t bt; + /* Initialize the backtrace, using the buffer in tdata to store it. */ + bt_init(&bt, cons_tdata->vec); + prof_backtrace(&bt); + prof_bt_t *cons_bt = &bt; + + /* We haven't destroyed tctx yet, so gctx should be good to read. */ + prof_bt_t *prod_bt = &tctx->gctx->bt; + + new_node->next = NULL; + new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid, + prod_thr_name); + new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid, + cons_thr_name); + new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt); + new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt); + new_node->alloc_time_ns = nstime_ns(&alloc_time); + new_node->free_time_ns = nstime_ns(&free_time); + new_node->usize = usize; + + if (log_alloc_first == NULL) { + log_alloc_first = new_node; + log_alloc_last = new_node; + } else { + log_alloc_last->next = new_node; + log_alloc_last = new_node; + } + +label_done: + malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx); +} + void -prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) { +prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, + prof_tctx_t *tctx) { malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); + assert(tctx->cnts.curobjs > 0); assert(tctx->cnts.curbytes >= usize); tctx->cnts.curobjs--; tctx->cnts.curbytes -= usize; + prof_try_log(tsd, ptr, usize, tctx); + if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) { prof_tctx_destroy(tsd, tctx); } else { @@ -871,15 +1139,12 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) { void prof_sample_threshold_update(prof_tdata_t *tdata) { #ifdef JEMALLOC_PROF - uint64_t r; - double u; - if (!config_prof) { return; } if (lg_prof_sample == 0) { - tdata->bytes_until_sample = 0; + tsd_bytes_until_sample_set(tsd_fetch(), 0); return; } @@ -901,11 +1166,16 @@ prof_sample_threshold_update(prof_tdata_t *tdata) { * pp 500 * (http://luc.devroye.org/rnbookindex.html) */ - r = prng_lg_range_u64(&tdata->prng_state, 53); - u = (double)r * (1.0/9007199254740992.0L); - tdata->bytes_until_sample = (uint64_t)(log(u) / + uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53); + double u = (double)r * (1.0/9007199254740992.0L); + uint64_t bytes_until_sample = (uint64_t)(log(u) / log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) + (uint64_t)1U; + if (bytes_until_sample > SSIZE_MAX) { + bytes_until_sample = SSIZE_MAX; + } + tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample); + #endif } @@ -1022,7 +1292,7 @@ prof_dump_write(bool propagate_err, const char *s) { } } - if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { + if (prof_dump_buf_end + slen - i <= PROF_DUMP_BUFSIZE) { /* Finish writing. */ n = slen - i; } else { @@ -1033,6 +1303,7 @@ prof_dump_write(bool propagate_err, const char *s) { prof_dump_buf_end += n; i += n; } + assert(i == slen); return false; } @@ -1887,6 +2158,33 @@ prof_bt_keycomp(const void *k1, const void *k2) { return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); } +static void +prof_bt_node_hash(const void *key, size_t r_hash[2]) { + const prof_bt_node_t *bt_node = (prof_bt_node_t *)key; + prof_bt_hash((void *)(&bt_node->bt), r_hash); +} + +static bool +prof_bt_node_keycomp(const void *k1, const void *k2) { + const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1; + const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2; + return prof_bt_keycomp((void *)(&bt_node1->bt), + (void *)(&bt_node2->bt)); +} + +static void +prof_thr_node_hash(const void *key, size_t r_hash[2]) { + const prof_thr_node_t *thr_node = (prof_thr_node_t *)key; + hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash); +} + +static bool +prof_thr_node_keycomp(const void *k1, const void *k2) { + const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1; + const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2; + return thr_node1->thr_uid == thr_node2->thr_uid; +} + static uint64_t prof_thr_uid_alloc(tsdn_t *tsdn) { uint64_t thr_uid; @@ -2119,6 +2417,368 @@ prof_active_set(tsdn_t *tsdn, bool active) { return prof_active_old; } +#ifdef JEMALLOC_JET +size_t +prof_log_bt_count(void) { + size_t cnt = 0; + prof_bt_node_t *node = log_bt_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +size_t +prof_log_alloc_count(void) { + size_t cnt = 0; + prof_alloc_node_t *node = log_alloc_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +size_t +prof_log_thr_count(void) { + size_t cnt = 0; + prof_thr_node_t *node = log_thr_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +bool +prof_log_is_logging(void) { + return prof_logging_state == prof_logging_state_started; +} + +bool +prof_log_rep_check(void) { + if (prof_logging_state == prof_logging_state_stopped + && log_tables_initialized) { + return true; + } + + if (log_bt_last != NULL && log_bt_last->next != NULL) { + return true; + } + if (log_thr_last != NULL && log_thr_last->next != NULL) { + return true; + } + if (log_alloc_last != NULL && log_alloc_last->next != NULL) { + return true; + } + + size_t bt_count = prof_log_bt_count(); + size_t thr_count = prof_log_thr_count(); + size_t alloc_count = prof_log_alloc_count(); + + + if (prof_logging_state == prof_logging_state_stopped) { + if (bt_count != 0 || thr_count != 0 || alloc_count || 0) { + return true; + } + } + + prof_alloc_node_t *node = log_alloc_first; + while (node != NULL) { + if (node->alloc_bt_ind >= bt_count) { + return true; + } + if (node->free_bt_ind >= bt_count) { + return true; + } + if (node->alloc_thr_ind >= thr_count) { + return true; + } + if (node->free_thr_ind >= thr_count) { + return true; + } + if (node->alloc_time_ns > node->free_time_ns) { + return true; + } + node = node->next; + } + + return false; +} + +void +prof_log_dummy_set(bool new_value) { + prof_log_dummy = new_value; +} +#endif + +bool +prof_log_start(tsdn_t *tsdn, const char *filename) { + if (!opt_prof || !prof_booted) { + return true; + } + + bool ret = false; + size_t buf_size = PATH_MAX + 1; + + malloc_mutex_lock(tsdn, &log_mtx); + + if (prof_logging_state != prof_logging_state_stopped) { + ret = true; + } else if (filename == NULL) { + /* Make default name. */ + malloc_snprintf(log_filename, buf_size, "%s.%d.%"FMTu64".json", + opt_prof_prefix, prof_getpid(), log_seq); + log_seq++; + prof_logging_state = prof_logging_state_started; + } else if (strlen(filename) >= buf_size) { + ret = true; + } else { + strcpy(log_filename, filename); + prof_logging_state = prof_logging_state_started; + } + + if (!ret) { + nstime_update(&log_start_timestamp); + } + + malloc_mutex_unlock(tsdn, &log_mtx); + + return ret; +} + +/* Used as an atexit function to stop logging on exit. */ +static void +prof_log_stop_final(void) { + tsd_t *tsd = tsd_fetch(); + prof_log_stop(tsd_tsdn(tsd)); +} + +struct prof_emitter_cb_arg_s { + int fd; + ssize_t ret; +}; + +static void +prof_emitter_write_cb(void *opaque, const char *to_write) { + struct prof_emitter_cb_arg_s *arg = + (struct prof_emitter_cb_arg_s *)opaque; + size_t bytes = strlen(to_write); +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + return; + } +#endif + arg->ret = write(arg->fd, (void *)to_write, bytes); +} + +/* + * prof_log_emit_{...} goes through the appropriate linked list, emitting each + * node to the json and deallocating it. + */ +static void +prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "threads"); + prof_thr_node_t *thr_node = log_thr_first; + prof_thr_node_t *thr_old_node; + while (thr_node != NULL) { + emitter_json_object_begin(emitter); + + emitter_json_kv(emitter, "thr_uid", emitter_type_uint64, + &thr_node->thr_uid); + + char *thr_name = thr_node->name; + + emitter_json_kv(emitter, "thr_name", emitter_type_string, + &thr_name); + + emitter_json_object_end(emitter); + thr_old_node = thr_node; + thr_node = thr_node->next; + idalloc(tsd, thr_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "stack_traces"); + prof_bt_node_t *bt_node = log_bt_first; + prof_bt_node_t *bt_old_node; + /* + * Calculate how many hex digits we need: twice number of bytes, two for + * "0x", and then one more for terminating '\0'. + */ + char buf[2 * sizeof(intptr_t) + 3]; + size_t buf_sz = sizeof(buf); + while (bt_node != NULL) { + emitter_json_array_begin(emitter); + size_t i; + for (i = 0; i < bt_node->bt.len; i++) { + malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]); + char *trace_str = buf; + emitter_json_value(emitter, emitter_type_string, + &trace_str); + } + emitter_json_array_end(emitter); + + bt_old_node = bt_node; + bt_node = bt_node->next; + idalloc(tsd, bt_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "allocations"); + prof_alloc_node_t *alloc_node = log_alloc_first; + prof_alloc_node_t *alloc_old_node; + while (alloc_node != NULL) { + emitter_json_object_begin(emitter); + + emitter_json_kv(emitter, "alloc_thread", emitter_type_size, + &alloc_node->alloc_thr_ind); + + emitter_json_kv(emitter, "free_thread", emitter_type_size, + &alloc_node->free_thr_ind); + + emitter_json_kv(emitter, "alloc_trace", emitter_type_size, + &alloc_node->alloc_bt_ind); + + emitter_json_kv(emitter, "free_trace", emitter_type_size, + &alloc_node->free_bt_ind); + + emitter_json_kv(emitter, "alloc_timestamp", + emitter_type_uint64, &alloc_node->alloc_time_ns); + + emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64, + &alloc_node->free_time_ns); + + emitter_json_kv(emitter, "usize", emitter_type_uint64, + &alloc_node->usize); + + emitter_json_object_end(emitter); + + alloc_old_node = alloc_node; + alloc_node = alloc_node->next; + idalloc(tsd, alloc_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_metadata(emitter_t *emitter) { + emitter_json_object_kv_begin(emitter, "info"); + + nstime_t now = NSTIME_ZERO_INITIALIZER; + + nstime_update(&now); + uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp); + emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns); + + char *vers = JEMALLOC_VERSION; + emitter_json_kv(emitter, "version", + emitter_type_string, &vers); + + emitter_json_kv(emitter, "lg_sample_rate", + emitter_type_int, &lg_prof_sample); + + int pid = prof_getpid(); + emitter_json_kv(emitter, "pid", emitter_type_int, &pid); + + emitter_json_object_end(emitter); +} + + +bool +prof_log_stop(tsdn_t *tsdn) { + if (!opt_prof || !prof_booted) { + return true; + } + + tsd_t *tsd = tsdn_tsd(tsdn); + malloc_mutex_lock(tsdn, &log_mtx); + + if (prof_logging_state != prof_logging_state_started) { + malloc_mutex_unlock(tsdn, &log_mtx); + return true; + } + + /* + * Set the state to dumping. We'll set it to stopped when we're done. + * Since other threads won't be able to start/stop/log when the state is + * dumping, we don't have to hold the lock during the whole method. + */ + prof_logging_state = prof_logging_state_dumping; + malloc_mutex_unlock(tsdn, &log_mtx); + + + emitter_t emitter; + + /* Create a file. */ + + int fd; +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + fd = 0; + } else { + fd = creat(log_filename, 0644); + } +#else + fd = creat(log_filename, 0644); +#endif + + if (fd == -1) { + malloc_printf(": creat() for log file \"%s\" " + " failed with %d\n", log_filename, errno); + if (opt_abort) { + abort(); + } + return true; + } + + /* Emit to json. */ + struct prof_emitter_cb_arg_s arg; + arg.fd = fd; + emitter_init(&emitter, emitter_output_json, &prof_emitter_write_cb, + (void *)(&arg)); + + emitter_begin(&emitter); + prof_log_emit_metadata(&emitter); + prof_log_emit_threads(tsd, &emitter); + prof_log_emit_traces(tsd, &emitter); + prof_log_emit_allocs(tsd, &emitter); + emitter_end(&emitter); + + /* Reset global state. */ + if (log_tables_initialized) { + ckh_delete(tsd, &log_bt_node_set); + ckh_delete(tsd, &log_thr_node_set); + } + log_tables_initialized = false; + log_bt_index = 0; + log_thr_index = 0; + log_bt_first = NULL; + log_bt_last = NULL; + log_thr_first = NULL; + log_thr_last = NULL; + log_alloc_first = NULL; + log_alloc_last = NULL; + + malloc_mutex_lock(tsdn, &log_mtx); + prof_logging_state = prof_logging_state_stopped; + malloc_mutex_unlock(tsdn, &log_mtx); + +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + return false; + } +#endif + return close(fd); +} + const char * prof_thread_name_get(tsd_t *tsd) { prof_tdata_t *tdata; @@ -2355,6 +3015,35 @@ prof_boot2(tsd_t *tsd) { } } + if (opt_prof_log) { + prof_log_start(tsd_tsdn(tsd), NULL); + } + + if (atexit(prof_log_stop_final) != 0) { + malloc_write(": Error in atexit() " + "for logging\n"); + if (opt_abort) { + abort(); + } + } + + if (malloc_mutex_init(&log_mtx, "prof_log", + WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) { + return true; + } + + if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, + prof_bt_node_hash, prof_bt_node_keycomp)) { + return true; + } + + if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, + prof_thr_node_hash, prof_thr_node_keycomp)) { + return true; + } + + log_tables_initialized = true; + gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), CACHELINE); @@ -2382,16 +3071,14 @@ prof_boot2(tsd_t *tsd) { return true; } } - } - #ifdef JEMALLOC_PROF_LIBGCC - /* - * Cause the backtracing machinery to allocate its internal state - * before enabling profiling. - */ - _Unwind_Backtrace(prof_unwind_init_callback, NULL); + /* + * Cause the backtracing machinery to allocate its internal + * state before enabling profiling. + */ + _Unwind_Backtrace(prof_unwind_init_callback, NULL); #endif - + } prof_booted = true; return false; diff --git a/deps/jemalloc/src/rtree.c b/deps/jemalloc/src/rtree.c index 53702cf72..4ae41fe2f 100644 --- a/deps/jemalloc/src/rtree.c +++ b/deps/jemalloc/src/rtree.c @@ -39,7 +39,7 @@ rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) { /* Nodes are never deleted during normal operation. */ not_reached(); } -UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = +rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = rtree_node_dalloc_impl; static rtree_leaf_elm_t * @@ -54,7 +54,7 @@ rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) { /* Leaves are never deleted during normal operation. */ not_reached(); } -UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = +rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = rtree_leaf_dalloc_impl; #ifdef JEMALLOC_JET diff --git a/deps/jemalloc/src/safety_check.c b/deps/jemalloc/src/safety_check.c new file mode 100644 index 000000000..804155dcf --- /dev/null +++ b/deps/jemalloc/src/safety_check.c @@ -0,0 +1,24 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +static void (*safety_check_abort)(const char *message); + +void safety_check_set_abort(void (*abort_fn)(const char *)) { + safety_check_abort = abort_fn; +} + +void safety_check_fail(const char *format, ...) { + char buf[MALLOC_PRINTF_BUFSIZE]; + + va_list ap; + va_start(ap, format); + malloc_vsnprintf(buf, MALLOC_PRINTF_BUFSIZE, format, ap); + va_end(ap); + + if (safety_check_abort == NULL) { + malloc_write(buf); + abort(); + } else { + safety_check_abort(buf); + } +} diff --git a/deps/jemalloc/src/sc.c b/deps/jemalloc/src/sc.c new file mode 100644 index 000000000..89ddb6ba6 --- /dev/null +++ b/deps/jemalloc/src/sc.c @@ -0,0 +1,313 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bit_util.h" +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/pages.h" +#include "jemalloc/internal/sc.h" + +/* + * This module computes the size classes used to satisfy allocations. The logic + * here was ported more or less line-by-line from a shell script, and because of + * that is not the most idiomatic C. Eventually we should fix this, but for now + * at least the damage is compartmentalized to this file. + */ + +sc_data_t sc_data_global; + +static size_t +reg_size_compute(int lg_base, int lg_delta, int ndelta) { + return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); +} + +/* Returns the number of pages in the slab. */ +static int +slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) { + size_t page = (ZU(1) << lg_page); + size_t reg_size = reg_size_compute(lg_base, lg_delta, ndelta); + + size_t try_slab_size = page; + size_t try_nregs = try_slab_size / reg_size; + size_t perfect_slab_size = 0; + bool perfect = false; + /* + * This loop continues until we find the least common multiple of the + * page size and size class size. Size classes are all of the form + * base + ndelta * delta == (ndelta + base/ndelta) * delta, which is + * (ndelta + ngroup) * delta. The way we choose slabbing strategies + * means that delta is at most the page size and ndelta < ngroup. So + * the loop executes for at most 2 * ngroup - 1 iterations, which is + * also the bound on the number of pages in a slab chosen by default. + * With the current default settings, this is at most 7. + */ + while (!perfect) { + perfect_slab_size = try_slab_size; + size_t perfect_nregs = try_nregs; + try_slab_size += page; + try_nregs = try_slab_size / reg_size; + if (perfect_slab_size == perfect_nregs * reg_size) { + perfect = true; + } + } + return (int)(perfect_slab_size / page); +} + +static void +size_class( + /* Output. */ + sc_t *sc, + /* Configuration decisions. */ + int lg_max_lookup, int lg_page, int lg_ngroup, + /* Inputs specific to the size class. */ + int index, int lg_base, int lg_delta, int ndelta) { + sc->index = index; + sc->lg_base = lg_base; + sc->lg_delta = lg_delta; + sc->ndelta = ndelta; + sc->psz = (reg_size_compute(lg_base, lg_delta, ndelta) + % (ZU(1) << lg_page) == 0); + size_t size = (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); + if (index == 0) { + assert(!sc->psz); + } + if (size < (ZU(1) << (lg_page + lg_ngroup))) { + sc->bin = true; + sc->pgs = slab_size(lg_page, lg_base, lg_delta, ndelta); + } else { + sc->bin = false; + sc->pgs = 0; + } + if (size <= (ZU(1) << lg_max_lookup)) { + sc->lg_delta_lookup = lg_delta; + } else { + sc->lg_delta_lookup = 0; + } +} + +static void +size_classes( + /* Output. */ + sc_data_t *sc_data, + /* Determined by the system. */ + size_t lg_ptr_size, int lg_quantum, + /* Configuration decisions. */ + int lg_tiny_min, int lg_max_lookup, int lg_page, int lg_ngroup) { + int ptr_bits = (1 << lg_ptr_size) * 8; + int ngroup = (1 << lg_ngroup); + int ntiny = 0; + int nlbins = 0; + int lg_tiny_maxclass = (unsigned)-1; + int nbins = 0; + int npsizes = 0; + + int index = 0; + + int ndelta = 0; + int lg_base = lg_tiny_min; + int lg_delta = lg_base; + + /* Outputs that we update as we go. */ + size_t lookup_maxclass = 0; + size_t small_maxclass = 0; + int lg_large_minclass = 0; + size_t large_maxclass = 0; + + /* Tiny size classes. */ + while (lg_base < lg_quantum) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + if (sc->lg_delta_lookup != 0) { + nlbins = index + 1; + } + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + ntiny++; + /* Final written value is correct. */ + lg_tiny_maxclass = lg_base; + index++; + lg_delta = lg_base; + lg_base++; + } + + /* First non-tiny (pseudo) group. */ + if (ntiny != 0) { + sc_t *sc = &sc_data->sc[index]; + /* + * See the note in sc.h; the first non-tiny size class has an + * unusual encoding. + */ + lg_base--; + ndelta = 1; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + index++; + lg_base++; + lg_delta++; + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + } + while (ndelta < ngroup) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + index++; + ndelta++; + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + } + + /* All remaining groups. */ + lg_base = lg_base + lg_ngroup; + while (lg_base < ptr_bits - 1) { + ndelta = 1; + int ndelta_limit; + if (lg_base == ptr_bits - 2) { + ndelta_limit = ngroup - 1; + } else { + ndelta_limit = ngroup; + } + while (ndelta <= ndelta_limit) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + if (sc->lg_delta_lookup != 0) { + nlbins = index + 1; + /* Final written value is correct. */ + lookup_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + } + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + /* Final written value is correct. */ + small_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + if (lg_ngroup > 0) { + lg_large_minclass = lg_base + 1; + } else { + lg_large_minclass = lg_base + 2; + } + } + large_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + index++; + ndelta++; + } + lg_base++; + lg_delta++; + } + /* Additional outputs. */ + int nsizes = index; + unsigned lg_ceil_nsizes = lg_ceil(nsizes); + + /* Fill in the output data. */ + sc_data->ntiny = ntiny; + sc_data->nlbins = nlbins; + sc_data->nbins = nbins; + sc_data->nsizes = nsizes; + sc_data->lg_ceil_nsizes = lg_ceil_nsizes; + sc_data->npsizes = npsizes; + sc_data->lg_tiny_maxclass = lg_tiny_maxclass; + sc_data->lookup_maxclass = lookup_maxclass; + sc_data->small_maxclass = small_maxclass; + sc_data->lg_large_minclass = lg_large_minclass; + sc_data->large_minclass = (ZU(1) << lg_large_minclass); + sc_data->large_maxclass = large_maxclass; + + /* + * We compute these values in two ways: + * - Incrementally, as above. + * - In macros, in sc.h. + * The computation is easier when done incrementally, but putting it in + * a constant makes it available to the fast paths without having to + * touch the extra global cacheline. We assert, however, that the two + * computations are equivalent. + */ + assert(sc_data->npsizes == SC_NPSIZES); + assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS); + assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS); + assert(sc_data->large_minclass == SC_LARGE_MINCLASS); + assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS); + assert(sc_data->large_maxclass == SC_LARGE_MAXCLASS); + + /* + * In the allocation fastpath, we want to assume that we can + * unconditionally subtract the requested allocation size from + * a ssize_t, and detect passing through 0 correctly. This + * results in optimal generated code. For this to work, the + * maximum allocation size must be less than SSIZE_MAX. + */ + assert(SC_LARGE_MAXCLASS < SSIZE_MAX); +} + +void +sc_data_init(sc_data_t *sc_data) { + assert(!sc_data->initialized); + + int lg_max_lookup = 12; + + size_classes(sc_data, LG_SIZEOF_PTR, LG_QUANTUM, SC_LG_TINY_MIN, + lg_max_lookup, LG_PAGE, 2); + + sc_data->initialized = true; +} + +static void +sc_data_update_sc_slab_size(sc_t *sc, size_t reg_size, size_t pgs_guess) { + size_t min_pgs = reg_size / PAGE; + if (reg_size % PAGE != 0) { + min_pgs++; + } + /* + * BITMAP_MAXBITS is actually determined by putting the smallest + * possible size-class on one page, so this can never be 0. + */ + size_t max_pgs = BITMAP_MAXBITS * reg_size / PAGE; + + assert(min_pgs <= max_pgs); + assert(min_pgs > 0); + assert(max_pgs >= 1); + if (pgs_guess < min_pgs) { + sc->pgs = (int)min_pgs; + } else if (pgs_guess > max_pgs) { + sc->pgs = (int)max_pgs; + } else { + sc->pgs = (int)pgs_guess; + } +} + +void +sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) { + assert(data->initialized); + for (int i = 0; i < data->nsizes; i++) { + sc_t *sc = &data->sc[i]; + if (!sc->bin) { + break; + } + size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta, + sc->ndelta); + if (begin <= reg_size && reg_size <= end) { + sc_data_update_sc_slab_size(sc, reg_size, pgs); + } + } +} + +void +sc_boot(sc_data_t *data) { + sc_data_init(data); +} diff --git a/deps/jemalloc/src/stats.c b/deps/jemalloc/src/stats.c index 08b9507cf..118e05d29 100644 --- a/deps/jemalloc/src/stats.c +++ b/deps/jemalloc/src/stats.c @@ -52,6 +52,20 @@ char opt_stats_print_opts[stats_print_tot_num_options+1] = ""; /******************************************************************************/ +static uint64_t +rate_per_second(uint64_t value, uint64_t uptime_ns) { + uint64_t billion = 1000000000; + if (uptime_ns == 0 || value == 0) { + return 0; + } + if (uptime_ns < billion) { + return value; + } else { + uint64_t uptime_s = uptime_ns / billion; + return value / uptime_s; + } +} + /* Calculate x.yyy and output a string (takes a fixed sized char array). */ static bool get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) { @@ -104,24 +118,26 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name, #define WIDTH_uint32_t 12 #define WIDTH_uint64_t 16 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ col = &col_##counter_type[k_##counter_type]; \ ++k_##counter_type; \ emitter_col_init(col, row); \ col->justify = emitter_justify_right; \ - col->width = WIDTH_##counter_type; \ + col->width = derived ? 8 : WIDTH_##counter_type; \ col->type = emitter_type_title; \ col->str_val = human; MUTEX_PROF_COUNTERS #undef OP #undef WIDTH_uint32_t #undef WIDTH_uint64_t + col_uint64_t[mutex_counter_total_wait_time_ps].width = 10; } static void mutex_stats_read_global(const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; col_name->str_val = name; @@ -129,12 +145,17 @@ mutex_stats_read_global(const char *name, emitter_col_t *col_name, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "mutexes", name, #counter); \ - CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "mutexes", name, #counter); \ + CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -145,7 +166,8 @@ static void mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; col_name->str_val = name; @@ -153,13 +175,17 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\ - CTL_M2_GET(cmd, arena_ind, \ - (counter_type *)&dst->bool_val, counter_type); + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\ + CTL_M2_GET(cmd, arena_ind, (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -169,19 +195,25 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, static void mutex_stats_read_arena_bin(unsigned arena_ind, unsigned bin_ind, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.bins.0","mutex", #counter); \ - CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ - (counter_type *)&dst->bool_val, counter_type); + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.bins.0","mutex", #counter); \ + CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ + (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -204,19 +236,38 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, type, human) \ - col = &col_##type[k_##type]; \ - ++k_##type; \ - emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ - (const void *)&col->bool_val); +#define OP(counter, type, human, derived, base_counter) \ + if (!derived) { \ + col = &col_##type[k_##type]; \ + ++k_##type; \ + emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ + (const void *)&col->bool_val); \ + } MUTEX_PROF_COUNTERS; #undef OP #undef EMITTER_TYPE_uint32_t #undef EMITTER_TYPE_uint64_t } +#define COL(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_t col_##column_name; \ + emitter_col_init(&col_##column_name, &row_name); \ + col_##column_name.justify = emitter_justify_##left_or_right; \ + col_##column_name.width = col_width; \ + col_##column_name.type = emitter_type_##etype; + +#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ + COL(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_t header_##column_name; \ + emitter_col_init(&header_##column_name, &header_##row_name); \ + header_##column_name.justify = emitter_justify_##left_or_right; \ + header_##column_name.width = col_width; \ + header_##column_name.type = emitter_type_title; \ + header_##column_name.str_val = human ? human : #column_name; + + static void -stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { +stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { size_t page; bool in_gap, in_gap_prev; unsigned nbins, j; @@ -230,43 +281,37 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { emitter_row_t row; emitter_row_init(&row); -#define COL(name, left_or_right, col_width, etype) \ - emitter_col_t col_##name; \ - emitter_col_init(&col_##name, &row); \ - col_##name.justify = emitter_justify_##left_or_right; \ - col_##name.width = col_width; \ - col_##name.type = emitter_type_##etype; \ - emitter_col_t header_col_##name; \ - emitter_col_init(&header_col_##name, &header_row); \ - header_col_##name.justify = emitter_justify_##left_or_right; \ - header_col_##name.width = col_width; \ - header_col_##name.type = emitter_type_title; \ - header_col_##name.str_val = #name; - COL(size, right, 20, size) - COL(ind, right, 4, unsigned) - COL(allocated, right, 13, uint64) - COL(nmalloc, right, 13, uint64) - COL(ndalloc, right, 13, uint64) - COL(nrequests, right, 13, uint64) - COL(curregs, right, 13, size) - COL(curslabs, right, 13, size) - COL(regs, right, 5, unsigned) - COL(pgs, right, 4, size) + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, allocated, NULL, right, 13, uint64) + COL_HDR(row, nmalloc, NULL, right, 13, uint64) + COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, ndalloc, NULL, right, 13, uint64) + COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nrequests, NULL, right, 13, uint64) + COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64) + COL_HDR(row, nshards, NULL, right, 9, unsigned) + COL_HDR(row, curregs, NULL, right, 13, size) + COL_HDR(row, curslabs, NULL, right, 13, size) + COL_HDR(row, nonfull_slabs, NULL, right, 15, size) + COL_HDR(row, regs, NULL, right, 5, unsigned) + COL_HDR(row, pgs, NULL, right, 4, size) /* To buffer a right- and left-justified column. */ - COL(justify_spacer, right, 1, title) - COL(util, right, 6, title) - COL(nfills, right, 13, uint64) - COL(nflushes, right, 13, uint64) - COL(nslabs, right, 13, uint64) - COL(nreslabs, right, 13, uint64) -#undef COL + COL_HDR(row, justify_spacer, NULL, right, 1, title) + COL_HDR(row, util, NULL, right, 6, title) + COL_HDR(row, nfills, NULL, right, 13, uint64) + COL_HDR(row, nfills_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nflushes, NULL, right, 13, uint64) + COL_HDR(row, nflushes_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nslabs, NULL, right, 13, uint64) + COL_HDR(row, nreslabs, NULL, right, 13, uint64) + COL_HDR(row, nreslabs_ps, "(#/sec)", right, 8, uint64) /* Don't want to actually print the name. */ - header_col_justify_spacer.str_val = " "; + header_justify_spacer.str_val = " "; col_justify_spacer.str_val = " "; - emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters]; emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters]; @@ -284,16 +329,17 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { * We print a "bins:" header as part of the table row; we need to adjust * the header size column to compensate. */ - header_col_size.width -=5; + header_size.width -=5; emitter_table_printf(emitter, "bins:"); emitter_table_row(emitter, &header_row); - emitter_json_arr_begin(emitter, "bins"); + emitter_json_array_kv_begin(emitter, "bins"); for (j = 0, in_gap = false; j < nbins; j++) { uint64_t nslabs; size_t reg_size, slab_size, curregs; size_t curslabs; - uint32_t nregs; + size_t nonfull_slabs; + uint32_t nregs, nshards; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t nreslabs; @@ -310,6 +356,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { CTL_M2_GET("arenas.bin.0.size", j, ®_size, size_t); CTL_M2_GET("arenas.bin.0.nregs", j, &nregs, uint32_t); CTL_M2_GET("arenas.bin.0.slab_size", j, &slab_size, size_t); + CTL_M2_GET("arenas.bin.0.nshards", j, &nshards, uint32_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.nmalloc", i, j, &nmalloc, uint64_t); @@ -327,13 +374,15 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { uint64_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.curslabs", i, j, &curslabs, size_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nonfull_slabs", i, j, &nonfull_slabs, + size_t); if (mutex) { mutex_stats_read_arena_bin(i, j, col_mutex64, - col_mutex32); + col_mutex32, uptime); } - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); emitter_json_kv(emitter, "nmalloc", emitter_type_uint64, &nmalloc); emitter_json_kv(emitter, "ndalloc", emitter_type_uint64, @@ -350,13 +399,15 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { &nreslabs); emitter_json_kv(emitter, "curslabs", emitter_type_size, &curslabs); + emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size, + &nonfull_slabs); if (mutex) { - emitter_json_dict_begin(emitter, "mutex"); + emitter_json_object_kv_begin(emitter, "mutex"); mutex_stats_emit(emitter, NULL, col_mutex64, col_mutex32); - emitter_json_dict_end(emitter); + emitter_json_object_end(emitter); } - emitter_json_arr_obj_end(emitter); + emitter_json_object_end(emitter); size_t availregs = nregs * curslabs; char util[6]; @@ -381,17 +432,25 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { col_ind.unsigned_val = j; col_allocated.size_val = curregs * reg_size; col_nmalloc.uint64_val = nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); col_ndalloc.uint64_val = ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; + col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); + col_nshards.unsigned_val = nshards; col_curregs.size_val = curregs; col_curslabs.size_val = curslabs; + col_nonfull_slabs.size_val = nonfull_slabs; col_regs.unsigned_val = nregs; col_pgs.size_val = slab_size / page; col_util.str_val = util; col_nfills.uint64_val = nfills; + col_nfills_ps.uint64_val = rate_per_second(nfills, uptime); col_nflushes.uint64_val = nflushes; + col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime); col_nslabs.uint64_val = nslabs; col_nreslabs.uint64_val = nreslabs; + col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime); /* * Note that mutex columns were initialized above, if mutex == @@ -400,7 +459,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { emitter_table_row(emitter, &row); } - emitter_json_arr_end(emitter); /* Close "bins". */ + emitter_json_array_end(emitter); /* Close "bins". */ if (in_gap) { emitter_table_printf(emitter, " ---\n"); @@ -408,7 +467,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { } static void -stats_arena_lextents_print(emitter_t *emitter, unsigned i) { +stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { unsigned nbins, nlextents, j; bool in_gap, in_gap_prev; @@ -420,34 +479,22 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i) { emitter_row_t row; emitter_row_init(&row); -#define COL(name, left_or_right, col_width, etype) \ - emitter_col_t header_##name; \ - emitter_col_init(&header_##name, &header_row); \ - header_##name.justify = emitter_justify_##left_or_right; \ - header_##name.width = col_width; \ - header_##name.type = emitter_type_title; \ - header_##name.str_val = #name; \ - \ - emitter_col_t col_##name; \ - emitter_col_init(&col_##name, &row); \ - col_##name.justify = emitter_justify_##left_or_right; \ - col_##name.width = col_width; \ - col_##name.type = emitter_type_##etype; - - COL(size, right, 20, size) - COL(ind, right, 4, unsigned) - COL(allocated, right, 13, size) - COL(nmalloc, right, 13, uint64) - COL(ndalloc, right, 13, uint64) - COL(nrequests, right, 13, uint64) - COL(curlextents, right, 13, size) -#undef COL + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, allocated, NULL, right, 13, size) + COL_HDR(row, nmalloc, NULL, right, 13, uint64) + COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, ndalloc, NULL, right, 13, uint64) + COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nrequests, NULL, right, 13, uint64) + COL_HDR(row, nrequests_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, curlextents, NULL, right, 13, size) /* As with bins, we label the large extents table. */ header_size.width -= 6; emitter_table_printf(emitter, "large:"); emitter_table_row(emitter, &header_row); - emitter_json_arr_begin(emitter, "lextents"); + emitter_json_array_kv_begin(emitter, "lextents"); for (j = 0, in_gap = false; j < nlextents; j++) { uint64_t nmalloc, ndalloc, nrequests; @@ -471,31 +518,122 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i) { CTL_M2_M4_GET("stats.arenas.0.lextents.0.curlextents", i, j, &curlextents, size_t); - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); emitter_json_kv(emitter, "curlextents", emitter_type_size, &curlextents); - emitter_json_arr_obj_end(emitter); + emitter_json_object_end(emitter); col_size.size_val = lextent_size; col_ind.unsigned_val = nbins + j; col_allocated.size_val = curlextents * lextent_size; col_nmalloc.uint64_val = nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); col_ndalloc.uint64_val = ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; + col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); col_curlextents.size_val = curlextents; if (!in_gap) { emitter_table_row(emitter, &row); } } - emitter_json_arr_end(emitter); /* Close "lextents". */ + emitter_json_array_end(emitter); /* Close "lextents". */ if (in_gap) { emitter_table_printf(emitter, " ---\n"); } } static void -stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind) { +stats_arena_extents_print(emitter_t *emitter, unsigned i) { + unsigned j; + bool in_gap, in_gap_prev; + emitter_row_t header_row; + emitter_row_init(&header_row); + emitter_row_t row; + emitter_row_init(&row); + + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, ndirty, NULL, right, 13, size) + COL_HDR(row, dirty, NULL, right, 13, size) + COL_HDR(row, nmuzzy, NULL, right, 13, size) + COL_HDR(row, muzzy, NULL, right, 13, size) + COL_HDR(row, nretained, NULL, right, 13, size) + COL_HDR(row, retained, NULL, right, 13, size) + COL_HDR(row, ntotal, NULL, right, 13, size) + COL_HDR(row, total, NULL, right, 13, size) + + /* Label this section. */ + header_size.width -= 8; + emitter_table_printf(emitter, "extents:"); + emitter_table_row(emitter, &header_row); + emitter_json_array_kv_begin(emitter, "extents"); + + in_gap = false; + for (j = 0; j < SC_NPSIZES; j++) { + size_t ndirty, nmuzzy, nretained, total, dirty_bytes, + muzzy_bytes, retained_bytes, total_bytes; + CTL_M2_M4_GET("stats.arenas.0.extents.0.ndirty", i, j, + &ndirty, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.nmuzzy", i, j, + &nmuzzy, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.nretained", i, j, + &nretained, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.dirty_bytes", i, j, + &dirty_bytes, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.muzzy_bytes", i, j, + &muzzy_bytes, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.retained_bytes", i, j, + &retained_bytes, size_t); + total = ndirty + nmuzzy + nretained; + total_bytes = dirty_bytes + muzzy_bytes + retained_bytes; + + in_gap_prev = in_gap; + in_gap = (total == 0); + + if (in_gap_prev && !in_gap) { + emitter_table_printf(emitter, + " ---\n"); + } + + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); + emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy); + emitter_json_kv(emitter, "nretained", emitter_type_size, + &nretained); + + emitter_json_kv(emitter, "dirty_bytes", emitter_type_size, + &dirty_bytes); + emitter_json_kv(emitter, "muzzy_bytes", emitter_type_size, + &muzzy_bytes); + emitter_json_kv(emitter, "retained_bytes", emitter_type_size, + &retained_bytes); + emitter_json_object_end(emitter); + + col_size.size_val = sz_pind2sz(j); + col_ind.size_val = j; + col_ndirty.size_val = ndirty; + col_dirty.size_val = dirty_bytes; + col_nmuzzy.size_val = nmuzzy; + col_muzzy.size_val = muzzy_bytes; + col_nretained.size_val = nretained; + col_retained.size_val = retained_bytes; + col_ntotal.size_val = total; + col_total.size_val = total_bytes; + + if (!in_gap) { + emitter_table_row(emitter, &row); + } + } + emitter_json_array_end(emitter); /* Close "extents". */ + if (in_gap) { + emitter_table_printf(emitter, " ---\n"); + } +} + +static void +stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { emitter_row_t row; emitter_col_t col_name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; @@ -504,36 +642,38 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind) { emitter_row_init(&row); mutex_stats_init_cols(&row, "", &col_name, col64, col32); - emitter_json_dict_begin(emitter, "mutexes"); + emitter_json_object_kv_begin(emitter, "mutexes"); emitter_table_row(emitter, &row); for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes; i++) { const char *name = arena_mutex_names[i]; - emitter_json_dict_begin(emitter, name); + emitter_json_object_kv_begin(emitter, name); mutex_stats_read_arena(arena_ind, i, name, &col_name, col64, - col32); + col32, uptime); mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_dict_end(emitter); /* Close the mutex dict. */ + emitter_json_object_end(emitter); /* Close the mutex dict. */ } - emitter_json_dict_end(emitter); /* End "mutexes". */ + emitter_json_object_end(emitter); /* End "mutexes". */ } static void stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, - bool mutex) { + bool mutex, bool extents) { unsigned nthreads; const char *dss; ssize_t dirty_decay_ms, muzzy_decay_ms; size_t page, pactive, pdirty, pmuzzy, mapped, retained; - size_t base, internal, resident, metadata_thp; + size_t base, internal, resident, metadata_thp, extent_avail; uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; size_t small_allocated; - uint64_t small_nmalloc, small_ndalloc, small_nrequests; + uint64_t small_nmalloc, small_ndalloc, small_nrequests, small_nfills, + small_nflushes; size_t large_allocated; - uint64_t large_nmalloc, large_ndalloc, large_nrequests; - size_t tcache_bytes; + uint64_t large_nmalloc, large_ndalloc, large_nrequests, large_nfills, + large_nflushes; + size_t tcache_bytes, abandoned_vm; uint64_t uptime; CTL_GET("arenas.page", &page, size_t); @@ -594,98 +734,74 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, &muzzy_purged); /* Table-style emission. */ - emitter_col_t decay_type; - emitter_col_init(&decay_type, &decay_row); - decay_type.justify = emitter_justify_right; - decay_type.width = 9; - decay_type.type = emitter_type_title; - decay_type.str_val = "decaying:"; + COL(decay_row, decay_type, right, 9, title); + col_decay_type.str_val = "decaying:"; - emitter_col_t decay_time; - emitter_col_init(&decay_time, &decay_row); - decay_time.justify = emitter_justify_right; - decay_time.width = 6; - decay_time.type = emitter_type_title; - decay_time.str_val = "time"; + COL(decay_row, decay_time, right, 6, title); + col_decay_time.str_val = "time"; - emitter_col_t decay_npages; - emitter_col_init(&decay_npages, &decay_row); - decay_npages.justify = emitter_justify_right; - decay_npages.width = 13; - decay_npages.type = emitter_type_title; - decay_npages.str_val = "npages"; + COL(decay_row, decay_npages, right, 13, title); + col_decay_npages.str_val = "npages"; - emitter_col_t decay_sweeps; - emitter_col_init(&decay_sweeps, &decay_row); - decay_sweeps.justify = emitter_justify_right; - decay_sweeps.width = 13; - decay_sweeps.type = emitter_type_title; - decay_sweeps.str_val = "sweeps"; + COL(decay_row, decay_sweeps, right, 13, title); + col_decay_sweeps.str_val = "sweeps"; - emitter_col_t decay_madvises; - emitter_col_init(&decay_madvises, &decay_row); - decay_madvises.justify = emitter_justify_right; - decay_madvises.width = 13; - decay_madvises.type = emitter_type_title; - decay_madvises.str_val = "madvises"; + COL(decay_row, decay_madvises, right, 13, title); + col_decay_madvises.str_val = "madvises"; - emitter_col_t decay_purged; - emitter_col_init(&decay_purged, &decay_row); - decay_purged.justify = emitter_justify_right; - decay_purged.width = 13; - decay_purged.type = emitter_type_title; - decay_purged.str_val = "purged"; + COL(decay_row, decay_purged, right, 13, title); + col_decay_purged.str_val = "purged"; /* Title row. */ emitter_table_row(emitter, &decay_row); /* Dirty row. */ - decay_type.str_val = "dirty:"; + col_decay_type.str_val = "dirty:"; if (dirty_decay_ms >= 0) { - decay_time.type = emitter_type_ssize; - decay_time.ssize_val = dirty_decay_ms; + col_decay_time.type = emitter_type_ssize; + col_decay_time.ssize_val = dirty_decay_ms; } else { - decay_time.type = emitter_type_title; - decay_time.str_val = "N/A"; + col_decay_time.type = emitter_type_title; + col_decay_time.str_val = "N/A"; } - decay_npages.type = emitter_type_size; - decay_npages.size_val = pdirty; + col_decay_npages.type = emitter_type_size; + col_decay_npages.size_val = pdirty; - decay_sweeps.type = emitter_type_uint64; - decay_sweeps.uint64_val = dirty_npurge; + col_decay_sweeps.type = emitter_type_uint64; + col_decay_sweeps.uint64_val = dirty_npurge; - decay_madvises.type = emitter_type_uint64; - decay_madvises.uint64_val = dirty_nmadvise; + col_decay_madvises.type = emitter_type_uint64; + col_decay_madvises.uint64_val = dirty_nmadvise; - decay_purged.type = emitter_type_uint64; - decay_purged.uint64_val = dirty_purged; + col_decay_purged.type = emitter_type_uint64; + col_decay_purged.uint64_val = dirty_purged; emitter_table_row(emitter, &decay_row); /* Muzzy row. */ - decay_type.str_val = "muzzy:"; + col_decay_type.str_val = "muzzy:"; if (muzzy_decay_ms >= 0) { - decay_time.type = emitter_type_ssize; - decay_time.ssize_val = muzzy_decay_ms; + col_decay_time.type = emitter_type_ssize; + col_decay_time.ssize_val = muzzy_decay_ms; } else { - decay_time.type = emitter_type_title; - decay_time.str_val = "N/A"; + col_decay_time.type = emitter_type_title; + col_decay_time.str_val = "N/A"; } - decay_npages.type = emitter_type_size; - decay_npages.size_val = pmuzzy; + col_decay_npages.type = emitter_type_size; + col_decay_npages.size_val = pmuzzy; - decay_sweeps.type = emitter_type_uint64; - decay_sweeps.uint64_val = muzzy_npurge; + col_decay_sweeps.type = emitter_type_uint64; + col_decay_sweeps.uint64_val = muzzy_npurge; - decay_madvises.type = emitter_type_uint64; - decay_madvises.uint64_val = muzzy_nmadvise; + col_decay_madvises.type = emitter_type_uint64; + col_decay_madvises.uint64_val = muzzy_nmadvise; - decay_purged.type = emitter_type_uint64; - decay_purged.uint64_val = muzzy_purged; + col_decay_purged.type = emitter_type_uint64; + col_decay_purged.uint64_val = muzzy_purged; emitter_table_row(emitter, &decay_row); @@ -693,81 +809,119 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, emitter_row_t alloc_count_row; emitter_row_init(&alloc_count_row); - emitter_col_t alloc_count_title; - emitter_col_init(&alloc_count_title, &alloc_count_row); - alloc_count_title.justify = emitter_justify_left; - alloc_count_title.width = 25; - alloc_count_title.type = emitter_type_title; - alloc_count_title.str_val = ""; + COL(alloc_count_row, count_title, left, 21, title); + col_count_title.str_val = ""; - emitter_col_t alloc_count_allocated; - emitter_col_init(&alloc_count_allocated, &alloc_count_row); - alloc_count_allocated.justify = emitter_justify_right; - alloc_count_allocated.width = 12; - alloc_count_allocated.type = emitter_type_title; - alloc_count_allocated.str_val = "allocated"; + COL(alloc_count_row, count_allocated, right, 16, title); + col_count_allocated.str_val = "allocated"; - emitter_col_t alloc_count_nmalloc; - emitter_col_init(&alloc_count_nmalloc, &alloc_count_row); - alloc_count_nmalloc.justify = emitter_justify_right; - alloc_count_nmalloc.width = 12; - alloc_count_nmalloc.type = emitter_type_title; - alloc_count_nmalloc.str_val = "nmalloc"; + COL(alloc_count_row, count_nmalloc, right, 16, title); + col_count_nmalloc.str_val = "nmalloc"; + COL(alloc_count_row, count_nmalloc_ps, right, 8, title); + col_count_nmalloc_ps.str_val = "(#/sec)"; - emitter_col_t alloc_count_ndalloc; - emitter_col_init(&alloc_count_ndalloc, &alloc_count_row); - alloc_count_ndalloc.justify = emitter_justify_right; - alloc_count_ndalloc.width = 12; - alloc_count_ndalloc.type = emitter_type_title; - alloc_count_ndalloc.str_val = "ndalloc"; + COL(alloc_count_row, count_ndalloc, right, 16, title); + col_count_ndalloc.str_val = "ndalloc"; + COL(alloc_count_row, count_ndalloc_ps, right, 8, title); + col_count_ndalloc_ps.str_val = "(#/sec)"; - emitter_col_t alloc_count_nrequests; - emitter_col_init(&alloc_count_nrequests, &alloc_count_row); - alloc_count_nrequests.justify = emitter_justify_right; - alloc_count_nrequests.width = 12; - alloc_count_nrequests.type = emitter_type_title; - alloc_count_nrequests.str_val = "nrequests"; + COL(alloc_count_row, count_nrequests, right, 16, title); + col_count_nrequests.str_val = "nrequests"; + COL(alloc_count_row, count_nrequests_ps, right, 10, title); + col_count_nrequests_ps.str_val = "(#/sec)"; + + COL(alloc_count_row, count_nfills, right, 16, title); + col_count_nfills.str_val = "nfill"; + COL(alloc_count_row, count_nfills_ps, right, 10, title); + col_count_nfills_ps.str_val = "(#/sec)"; + + COL(alloc_count_row, count_nflushes, right, 16, title); + col_count_nflushes.str_val = "nflush"; + COL(alloc_count_row, count_nflushes_ps, right, 10, title); + col_count_nflushes_ps.str_val = "(#/sec)"; emitter_table_row(emitter, &alloc_count_row); + col_count_nmalloc_ps.type = emitter_type_uint64; + col_count_ndalloc_ps.type = emitter_type_uint64; + col_count_nrequests_ps.type = emitter_type_uint64; + col_count_nfills_ps.type = emitter_type_uint64; + col_count_nflushes_ps.type = emitter_type_uint64; + #define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ &small_or_large##_##name, valtype##_t); \ emitter_json_kv(emitter, #name, emitter_type_##valtype, \ &small_or_large##_##name); \ - alloc_count_##name.type = emitter_type_##valtype; \ - alloc_count_##name.valtype##_val = small_or_large##_##name; + col_count_##name.type = emitter_type_##valtype; \ + col_count_##name.valtype##_val = small_or_large##_##name; - emitter_json_dict_begin(emitter, "small"); - alloc_count_title.str_val = "small:"; + emitter_json_object_kv_begin(emitter, "small"); + col_count_title.str_val = "small:"; GET_AND_EMIT_ALLOC_STAT(small, allocated, size) GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64) + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64) + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64) + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(small, nfills, uint64) + col_count_nfills_ps.uint64_val = + rate_per_second(col_count_nfills.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(small, nflushes, uint64) + col_count_nflushes_ps.uint64_val = + rate_per_second(col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); - emitter_json_dict_end(emitter); /* Close "small". */ + emitter_json_object_end(emitter); /* Close "small". */ - emitter_json_dict_begin(emitter, "large"); - alloc_count_title.str_val = "large:"; + emitter_json_object_kv_begin(emitter, "large"); + col_count_title.str_val = "large:"; GET_AND_EMIT_ALLOC_STAT(large, allocated, size) GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64) + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64) + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64) + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(large, nfills, uint64) + col_count_nfills_ps.uint64_val = + rate_per_second(col_count_nfills.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(large, nflushes, uint64) + col_count_nflushes_ps.uint64_val = + rate_per_second(col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); - emitter_json_dict_end(emitter); /* Close "large". */ + emitter_json_object_end(emitter); /* Close "large". */ #undef GET_AND_EMIT_ALLOC_STAT /* Aggregated small + large stats are emitter only in table mode. */ - alloc_count_title.str_val = "total:"; - alloc_count_allocated.size_val = small_allocated + large_allocated; - alloc_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; - alloc_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; - alloc_count_nrequests.uint64_val = small_nrequests + large_nrequests; + col_count_title.str_val = "total:"; + col_count_allocated.size_val = small_allocated + large_allocated; + col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; + col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; + col_count_nrequests.uint64_val = small_nrequests + large_nrequests; + col_count_nfills.uint64_val = small_nfills + large_nfills; + col_count_nflushes.uint64_val = small_nflushes + large_nflushes; + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); + col_count_nfills_ps.uint64_val = + rate_per_second(col_count_nfills.uint64_val, uptime); + col_count_nflushes_ps.uint64_val = + rate_per_second(col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); emitter_row_t mem_count_row; @@ -776,14 +930,14 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, emitter_col_t mem_count_title; emitter_col_init(&mem_count_title, &mem_count_row); mem_count_title.justify = emitter_justify_left; - mem_count_title.width = 25; + mem_count_title.width = 21; mem_count_title.type = emitter_type_title; mem_count_title.str_val = ""; emitter_col_t mem_count_val; emitter_col_init(&mem_count_val, &mem_count_row); mem_count_val.justify = emitter_justify_right; - mem_count_val.width = 12; + mem_count_val.width = 16; mem_count_val.type = emitter_type_title; mem_count_val.str_val = ""; @@ -809,16 +963,21 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, GET_AND_EMIT_MEM_STAT(metadata_thp) GET_AND_EMIT_MEM_STAT(tcache_bytes) GET_AND_EMIT_MEM_STAT(resident) + GET_AND_EMIT_MEM_STAT(abandoned_vm) + GET_AND_EMIT_MEM_STAT(extent_avail) #undef GET_AND_EMIT_MEM_STAT if (mutex) { - stats_arena_mutexes_print(emitter, i); + stats_arena_mutexes_print(emitter, i, uptime); } if (bins) { - stats_arena_bins_print(emitter, mutex, i); + stats_arena_bins_print(emitter, mutex, i, uptime); } if (large) { - stats_arena_lextents_print(emitter, i); + stats_arena_lextents_print(emitter, i, uptime); + } + if (extents) { + stats_arena_extents_print(emitter, i); } } @@ -857,6 +1016,7 @@ stats_general_print(emitter_t *emitter) { emitter_kv(emitter, "malloc_conf", "config.malloc_conf", emitter_type_string, &config_malloc_conf); + CONFIG_WRITE_BOOL(opt_safety_checks); CONFIG_WRITE_BOOL(prof); CONFIG_WRITE_BOOL(prof_libgcc); CONFIG_WRITE_BOOL(prof_libunwind); @@ -891,6 +1051,8 @@ stats_general_print(emitter_t *emitter) { #define OPT_WRITE_UNSIGNED(name) \ OPT_WRITE(name, uv, usz, emitter_type_unsigned) +#define OPT_WRITE_SIZE_T(name) \ + OPT_WRITE(name, sv, ssz, emitter_type_size) #define OPT_WRITE_SSIZE_T(name) \ OPT_WRITE(name, ssv, sssz, emitter_type_ssize) #define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ @@ -904,15 +1066,17 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("abort") OPT_WRITE_BOOL("abort_conf") + OPT_WRITE_BOOL("confirm_conf") OPT_WRITE_BOOL("retain") OPT_WRITE_CHAR_P("dss") OPT_WRITE_UNSIGNED("narenas") OPT_WRITE_CHAR_P("percpu_arena") + OPT_WRITE_SIZE_T("oversize_threshold") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms") OPT_WRITE_SSIZE_T_MUTABLE("muzzy_decay_ms", "arenas.muzzy_decay_ms") - OPT_WRITE_UNSIGNED("lg_extent_max_active_fit") + OPT_WRITE_SIZE_T("lg_extent_max_active_fit") OPT_WRITE_CHAR_P("junk") OPT_WRITE_BOOL("zero") OPT_WRITE_BOOL("utrace") @@ -977,7 +1141,7 @@ stats_general_print(emitter_t *emitter) { * The json output sticks arena info into an "arenas" dict; the table * output puts them at the top-level. */ - emitter_json_dict_begin(emitter, "arenas"); + emitter_json_object_kv_begin(emitter, "arenas"); CTL_GET("arenas.narenas", &uv, unsigned); emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv); @@ -1018,9 +1182,9 @@ stats_general_print(emitter_t *emitter) { * (not just omit the printing). */ if (emitter->output == emitter_output_json) { - emitter_json_arr_begin(emitter, "bin"); + emitter_json_array_kv_begin(emitter, "bin"); for (unsigned i = 0; i < nbins; i++) { - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); CTL_M2_GET("arenas.bin.0.size", i, &sv, size_t); emitter_json_kv(emitter, "size", emitter_type_size, @@ -1034,9 +1198,13 @@ stats_general_print(emitter_t *emitter) { emitter_json_kv(emitter, "slab_size", emitter_type_size, &sv); - emitter_json_arr_obj_end(emitter); + CTL_M2_GET("arenas.bin.0.nshards", i, &u32v, uint32_t); + emitter_json_kv(emitter, "nshards", emitter_type_uint32, + &u32v); + + emitter_json_object_end(emitter); } - emitter_json_arr_end(emitter); /* Close "bin". */ + emitter_json_array_end(emitter); /* Close "bin". */ } unsigned nlextents; @@ -1045,25 +1213,25 @@ stats_general_print(emitter_t *emitter) { emitter_type_unsigned, &nlextents); if (emitter->output == emitter_output_json) { - emitter_json_arr_begin(emitter, "lextent"); + emitter_json_array_kv_begin(emitter, "lextent"); for (unsigned i = 0; i < nlextents; i++) { - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); CTL_M2_GET("arenas.lextent.0.size", i, &sv, size_t); emitter_json_kv(emitter, "size", emitter_type_size, &sv); - emitter_json_arr_obj_end(emitter); + emitter_json_object_end(emitter); } - emitter_json_arr_end(emitter); /* Close "lextent". */ + emitter_json_array_end(emitter); /* Close "lextent". */ } - emitter_json_dict_end(emitter); /* Close "arenas" */ + emitter_json_object_end(emitter); /* Close "arenas" */ } static void stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, - bool unmerged, bool bins, bool large, bool mutex) { + bool unmerged, bool bins, bool large, bool mutex, bool extents) { /* * These should be deleted. We keep them around for a while, to aid in * the transition to the emitter code. @@ -1095,7 +1263,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, } /* Generic global stats. */ - emitter_json_dict_begin(emitter, "stats"); + emitter_json_object_kv_begin(emitter, "stats"); emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated); emitter_json_kv(emitter, "active", emitter_type_size, &active); emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata); @@ -1111,14 +1279,14 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, resident, mapped, retained); /* Background thread stats. */ - emitter_json_dict_begin(emitter, "background_thread"); + emitter_json_object_kv_begin(emitter, "background_thread"); emitter_json_kv(emitter, "num_threads", emitter_type_size, &num_background_threads); emitter_json_kv(emitter, "num_runs", emitter_type_uint64, &background_thread_num_runs); emitter_json_kv(emitter, "run_interval", emitter_type_uint64, &background_thread_run_interval); - emitter_json_dict_end(emitter); /* Close "background_thread". */ + emitter_json_object_end(emitter); /* Close "background_thread". */ emitter_table_printf(emitter, "Background threads: %zu, " "num_runs: %"FMTu64", run_interval: %"FMTu64" ns\n", @@ -1130,30 +1298,33 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, emitter_col_t name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; emitter_col_t col32[mutex_prof_num_uint32_t_counters]; + uint64_t uptime; emitter_row_init(&row); mutex_stats_init_cols(&row, "", &name, col64, col32); emitter_table_row(emitter, &row); - emitter_json_dict_begin(emitter, "mutexes"); + emitter_json_object_kv_begin(emitter, "mutexes"); + + CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { mutex_stats_read_global(global_mutex_names[i], &name, - col64, col32); - emitter_json_dict_begin(emitter, global_mutex_names[i]); + col64, col32, uptime); + emitter_json_object_kv_begin(emitter, global_mutex_names[i]); mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_dict_end(emitter); + emitter_json_object_end(emitter); } - emitter_json_dict_end(emitter); /* Close "mutexes". */ + emitter_json_object_end(emitter); /* Close "mutexes". */ } - emitter_json_dict_end(emitter); /* Close "stats". */ + emitter_json_object_end(emitter); /* Close "stats". */ if (merged || destroyed || unmerged) { unsigned narenas; - emitter_json_dict_begin(emitter, "stats.arenas"); + emitter_json_object_kv_begin(emitter, "stats.arenas"); CTL_GET("arenas.narenas", &narenas, unsigned); size_t mib[3]; @@ -1182,10 +1353,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, if (merged && (ninitialized > 1 || !unmerged)) { /* Print merged arena stats. */ emitter_table_printf(emitter, "Merged arenas stats:\n"); - emitter_json_dict_begin(emitter, "merged"); + emitter_json_object_kv_begin(emitter, "merged"); stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins, - large, mutex); - emitter_json_dict_end(emitter); /* Close "merged". */ + large, mutex, extents); + emitter_json_object_end(emitter); /* Close "merged". */ } /* Destroyed stats. */ @@ -1193,10 +1364,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, /* Print destroyed arena stats. */ emitter_table_printf(emitter, "Destroyed arenas stats:\n"); - emitter_json_dict_begin(emitter, "destroyed"); + emitter_json_object_kv_begin(emitter, "destroyed"); stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, - bins, large, mutex); - emitter_json_dict_end(emitter); /* Close "destroyed". */ + bins, large, mutex, extents); + emitter_json_object_end(emitter); /* Close "destroyed". */ } /* Unmerged stats. */ @@ -1206,18 +1377,18 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, char arena_ind_str[20]; malloc_snprintf(arena_ind_str, sizeof(arena_ind_str), "%u", i); - emitter_json_dict_begin(emitter, + emitter_json_object_kv_begin(emitter, arena_ind_str); emitter_table_printf(emitter, "arenas[%s]:\n", arena_ind_str); stats_arena_print(emitter, i, bins, - large, mutex); + large, mutex, extents); /* Close "". */ - emitter_json_dict_end(emitter); + emitter_json_object_end(emitter); } } } - emitter_json_dict_end(emitter); /* Close "stats.arenas". */ + emitter_json_object_end(emitter); /* Close "stats.arenas". */ } } @@ -1270,17 +1441,17 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cbopaque); emitter_begin(&emitter); emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); - emitter_json_dict_begin(&emitter, "jemalloc"); + emitter_json_object_kv_begin(&emitter, "jemalloc"); if (general) { stats_general_print(&emitter); } if (config_stats) { stats_print_helper(&emitter, merged, destroyed, unmerged, - bins, large, mutex); + bins, large, mutex, extents); } - emitter_json_dict_end(&emitter); /* Closes the "jemalloc" dict. */ + emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); emitter_end(&emitter); } diff --git a/deps/jemalloc/src/sz.c b/deps/jemalloc/src/sz.c index 9de77e45f..8633fb050 100644 --- a/deps/jemalloc/src/sz.c +++ b/deps/jemalloc/src/sz.c @@ -2,106 +2,63 @@ #include "jemalloc/internal/sz.h" JEMALLOC_ALIGNED(CACHELINE) -const size_t sz_pind2sz_tab[NPSIZES+1] = { -#define PSZ_yes(lg_grp, ndelta, lg_delta) \ - (((ZU(1)<sc[i]; + if (sc->psz) { + sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + pind++; + } + } + for (int i = pind; i <= (int)SC_NPSIZES; i++) { + sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE; + } +} JEMALLOC_ALIGNED(CACHELINE) -const size_t sz_index2size_tab[NSIZES] = { -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ - ((ZU(1)<sc[i]; + sz_index2size_tab[i] = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << (sc->lg_delta)); + } +} + +/* + * To keep this table small, we divide sizes by the tiny min size, which gives + * the smallest interval for which the result can change. + */ JEMALLOC_ALIGNED(CACHELINE) -const uint8_t sz_size2index_tab[] = { -#if LG_TINY_MIN == 0 -/* The div module doesn't support division by 1. */ -#error "Unsupported LG_TINY_MIN" -#define S2B_0(i) i, -#elif LG_TINY_MIN == 1 -#warning "Dangerous LG_TINY_MIN" -#define S2B_1(i) i, -#elif LG_TINY_MIN == 2 -#warning "Dangerous LG_TINY_MIN" -#define S2B_2(i) i, -#elif LG_TINY_MIN == 3 -#define S2B_3(i) i, -#elif LG_TINY_MIN == 4 -#define S2B_4(i) i, -#elif LG_TINY_MIN == 5 -#define S2B_5(i) i, -#elif LG_TINY_MIN == 6 -#define S2B_6(i) i, -#elif LG_TINY_MIN == 7 -#define S2B_7(i) i, -#elif LG_TINY_MIN == 8 -#define S2B_8(i) i, -#elif LG_TINY_MIN == 9 -#define S2B_9(i) i, -#elif LG_TINY_MIN == 10 -#define S2B_10(i) i, -#elif LG_TINY_MIN == 11 -#define S2B_11(i) i, -#else -#error "Unsupported LG_TINY_MIN" -#endif -#if LG_TINY_MIN < 1 -#define S2B_1(i) S2B_0(i) S2B_0(i) -#endif -#if LG_TINY_MIN < 2 -#define S2B_2(i) S2B_1(i) S2B_1(i) -#endif -#if LG_TINY_MIN < 3 -#define S2B_3(i) S2B_2(i) S2B_2(i) -#endif -#if LG_TINY_MIN < 4 -#define S2B_4(i) S2B_3(i) S2B_3(i) -#endif -#if LG_TINY_MIN < 5 -#define S2B_5(i) S2B_4(i) S2B_4(i) -#endif -#if LG_TINY_MIN < 6 -#define S2B_6(i) S2B_5(i) S2B_5(i) -#endif -#if LG_TINY_MIN < 7 -#define S2B_7(i) S2B_6(i) S2B_6(i) -#endif -#if LG_TINY_MIN < 8 -#define S2B_8(i) S2B_7(i) S2B_7(i) -#endif -#if LG_TINY_MIN < 9 -#define S2B_9(i) S2B_8(i) S2B_8(i) -#endif -#if LG_TINY_MIN < 10 -#define S2B_10(i) S2B_9(i) S2B_9(i) -#endif -#if LG_TINY_MIN < 11 -#define S2B_11(i) S2B_10(i) S2B_10(i) -#endif -#define S2B_no(i) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ - S2B_##lg_delta_lookup(index) - SIZE_CLASSES -#undef S2B_3 -#undef S2B_4 -#undef S2B_5 -#undef S2B_6 -#undef S2B_7 -#undef S2B_8 -#undef S2B_9 -#undef S2B_10 -#undef S2B_11 -#undef S2B_no -#undef SC -}; +uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1]; + +static void +sz_boot_size2index_tab(const sc_data_t *sc_data) { + size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1; + size_t dst_ind = 0; + for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max; + sc_ind++) { + const sc_t *sc = &sc_data->sc[sc_ind]; + size_t sz = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1) + >> SC_LG_TINY_MIN); + for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) { + sz_size2index_tab[dst_ind] = sc_ind; + } + } +} + +void +sz_boot(const sc_data_t *sc_data) { + sz_boot_pind2sz_tab(sc_data); + sz_boot_index2size_tab(sc_data); + sz_boot_size2index_tab(sc_data); +} diff --git a/deps/jemalloc/src/tcache.c b/deps/jemalloc/src/tcache.c index a769a6b17..50099a9f2 100644 --- a/deps/jemalloc/src/tcache.c +++ b/deps/jemalloc/src/tcache.c @@ -4,7 +4,8 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/safety_check.h" +#include "jemalloc/internal/sc.h" /******************************************************************************/ /* Data. */ @@ -41,7 +42,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { szind_t binind = tcache->next_gc_bin; cache_bin_t *tbin; - if (binind < NBINS) { + if (binind < SC_NBINS) { tbin = tcache_small_bin_get(tcache, binind); } else { tbin = tcache_large_bin_get(tcache, binind); @@ -50,7 +51,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { /* * Flush (ceiling) 3/4 of the objects below the low water mark. */ - if (binind < NBINS) { + if (binind < SC_NBINS) { tcache_bin_flush_small(tsd, tcache, tbin, binind, tbin->ncached - tbin->low_water + (tbin->low_water >> 2)); @@ -72,7 +73,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { * Increase fill count by 2X for small bins. Make sure * lg_fill_div stays greater than 0. */ - if (binind < NBINS && tcache->lg_fill_div[binind] > 1) { + if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) { tcache->lg_fill_div[binind]--; } } @@ -100,28 +101,67 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, return ret; } +/* Enabled with --enable-extra-size-check. */ +static void +tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind, + size_t nflush, extent_t **extents){ + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + /* + * Verify that the items in the tcache all have the correct size; this + * is useful for catching sized deallocation bugs, also to fail early + * instead of corrupting metadata. Since this can be turned on for opt + * builds, avoid the branch in the loop. + */ + szind_t szind; + size_t sz_sum = binind * nflush; + for (unsigned i = 0 ; i < nflush; i++) { + rtree_extent_szind_read(tsdn, &extents_rtree, + rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true, + &extents[i], &szind); + sz_sum -= szind; + } + if (sz_sum != 0) { + safety_check_fail(": size mismatch in thread cache " + "detected, likely caused by sized deallocation bugs by " + "application. Abort.\n"); + abort(); + } +} + void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, unsigned rem) { bool merged_stats = false; - assert(binind < NBINS); + assert(binind < SC_NBINS); assert((cache_bin_sz_t)rem <= tbin->ncached); arena_t *arena = tcache->arena; assert(arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); - /* Look up extent once per item. */ - for (unsigned i = 0 ; i < nflush; i++) { - item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); - } + /* Look up extent once per item. */ + if (config_opt_safety_checks) { + tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, + nflush, item_extent); + } else { + for (unsigned i = 0 ; i < nflush; i++) { + item_extent[i] = iealloc(tsd_tsdn(tsd), + *(tbin->avail - 1 - i)); + } + } while (nflush > 0) { /* Lock the arena bin associated with the first object. */ extent_t *extent = item_extent[0]; - arena_t *bin_arena = extent_arena_get(extent); - bin_t *bin = &bin_arena->bins[binind]; + unsigned bin_arena_ind = extent_arena_ind_get(extent); + arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind, + false); + unsigned binshard = extent_binshard_get(extent); + assert(binshard < bin_infos[binind].n_shards); + bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard]; if (config_prof && bin_arena == arena) { if (arena_prof_accum(tsd_tsdn(tsd), arena, @@ -132,8 +172,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, } malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (config_stats && bin_arena == arena) { - assert(!merged_stats); + if (config_stats && bin_arena == arena && !merged_stats) { merged_stats = true; bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; @@ -145,9 +184,10 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_get(extent) == bin_arena) { + if (extent_arena_ind_get(extent) == bin_arena_ind + && extent_binshard_get(extent) == binshard) { arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), - bin_arena, extent, ptr); + bin_arena, bin, binind, extent, ptr); } else { /* * This object was allocated via a different @@ -169,8 +209,9 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - bin_t *bin = &arena->bins[binind]; - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind, + &binshard); bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; @@ -193,50 +234,63 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, assert(binind < nhbins); assert((cache_bin_sz_t)rem <= tbin->ncached); - arena_t *arena = tcache->arena; - assert(arena != NULL); + arena_t *tcache_arena = tcache->arena; + assert(tcache_arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); + +#ifndef JEMALLOC_EXTRA_SIZE_CHECK /* Look up extent once per item. */ for (unsigned i = 0 ; i < nflush; i++) { item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); } - +#else + tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, + item_extent); +#endif while (nflush > 0) { /* Lock the arena associated with the first object. */ extent_t *extent = item_extent[0]; - arena_t *locked_arena = extent_arena_get(extent); - UNUSED bool idump; + unsigned locked_arena_ind = extent_arena_ind_get(extent); + arena_t *locked_arena = arena_get(tsd_tsdn(tsd), + locked_arena_ind, false); + bool idump; if (config_prof) { idump = false; } - malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); + bool lock_large = !arena_is_auto(locked_arena); + if (lock_large) { + malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); + } for (unsigned i = 0; i < nflush; i++) { void *ptr = *(tbin->avail - 1 - i); assert(ptr != NULL); extent = item_extent[i]; - if (extent_arena_get(extent) == locked_arena) { + if (extent_arena_ind_get(extent) == locked_arena_ind) { large_dalloc_prep_junked_locked(tsd_tsdn(tsd), extent); } } - if ((config_prof || config_stats) && locked_arena == arena) { + if ((config_prof || config_stats) && + (locked_arena == tcache_arena)) { if (config_prof) { - idump = arena_prof_accum(tsd_tsdn(tsd), arena, - tcache->prof_accumbytes); + idump = arena_prof_accum(tsd_tsdn(tsd), + tcache_arena, tcache->prof_accumbytes); tcache->prof_accumbytes = 0; } if (config_stats) { merged_stats = true; - arena_stats_large_nrequests_add(tsd_tsdn(tsd), - &arena->stats, binind, + arena_stats_large_flush_nrequests_add( + tsd_tsdn(tsd), &tcache_arena->stats, binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } } - malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); + if (lock_large) { + malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); + } unsigned ndeferred = 0; for (unsigned i = 0; i < nflush; i++) { @@ -244,7 +298,7 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_get(extent) == locked_arena) { + if (extent_arena_ind_get(extent) == locked_arena_ind) { large_dalloc_finish(tsd_tsdn(tsd), extent); } else { /* @@ -270,8 +324,8 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats, - binind, tbin->tstats.nrequests); + arena_stats_large_flush_nrequests_add(tsd_tsdn(tsd), + &tcache_arena->stats, binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } @@ -363,10 +417,10 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { size_t stack_offset = 0; assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); - memset(tcache->bins_small, 0, sizeof(cache_bin_t) * NBINS); - memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - NBINS)); + memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS); + memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS)); unsigned i = 0; - for (; i < NBINS; i++) { + for (; i < SC_NBINS; i++) { tcache->lg_fill_div[i] = 1; stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); /* @@ -458,7 +512,7 @@ static void tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tcache->arena != NULL); - for (unsigned i = 0; i < NBINS; i++) { + for (unsigned i = 0; i < SC_NBINS; i++) { cache_bin_t *tbin = tcache_small_bin_get(tcache, i); tcache_bin_flush_small(tsd, tcache, tbin, i, 0); @@ -466,7 +520,7 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tbin->tstats.nrequests == 0); } } - for (unsigned i = NBINS; i < nhbins; i++) { + for (unsigned i = SC_NBINS; i < nhbins; i++) { cache_bin_t *tbin = tcache_large_bin_get(tcache, i); tcache_bin_flush_large(tsd, tbin, i, 0, tcache); @@ -491,6 +545,7 @@ tcache_flush(tsd_t *tsd) { static void tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { tcache_flush_cache(tsd, tcache); + arena_t *arena = tcache->arena; tcache_arena_dissociate(tsd_tsdn(tsd), tcache); if (tsd_tcache) { @@ -503,6 +558,23 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { /* Release both the tcache struct and avail array. */ idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true); } + + /* + * The deallocation and tcache flush above may not trigger decay since + * we are on the tcache shutdown path (potentially with non-nominal + * tsd). Manually trigger decay to avoid pathological cases. Also + * include arena 0 because the tcache array is allocated from it. + */ + arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), + false, false); + + if (arena_nthreads_get(arena, false) == 0 && + !background_thread_enabled()) { + /* Force purging when no threads assigned to the arena anymore. */ + arena_decay(tsd_tsdn(tsd), arena, false, true); + } else { + arena_decay(tsd_tsdn(tsd), arena, false, false); + } } /* For auto tcache (embedded in TSD) only. */ @@ -532,10 +604,10 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { cassert(config_stats); /* Merge and reset tcache stats. */ - for (i = 0; i < NBINS; i++) { - bin_t *bin = &arena->bins[i]; + for (i = 0; i < SC_NBINS; i++) { cache_bin_t *tbin = tcache_small_bin_get(tcache, i); - malloc_mutex_lock(tsdn, &bin->lock); + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard); bin->stats.nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(tsdn, &bin->lock); tbin->tstats.nrequests = 0; @@ -543,7 +615,7 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { for (; i < nhbins; i++) { cache_bin_t *tbin = tcache_large_bin_get(tcache, i); - arena_stats_large_nrequests_add(tsdn, &arena->stats, i, + arena_stats_large_flush_nrequests_add(tsdn, &arena->stats, i, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } @@ -614,23 +686,32 @@ label_return: } static tcache_t * -tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) { +tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); if (elm->tcache == NULL) { return NULL; } tcache_t *tcache = elm->tcache; - elm->tcache = NULL; + if (allow_reinit) { + elm->tcache = TCACHES_ELM_NEED_REINIT; + } else { + elm->tcache = NULL; + } + + if (tcache == TCACHES_ELM_NEED_REINIT) { + return NULL; + } return tcache; } void tcaches_flush(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); - tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); + tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true); malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); if (tcache != NULL) { + /* Destroy the tcache; recreate in tcaches_get() if needed. */ tcache_destroy(tsd, tcache, false); } } @@ -639,7 +720,7 @@ void tcaches_destroy(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_t *elm = &tcaches[ind]; - tcache_t *tcache = tcaches_elm_remove(tsd, elm); + tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); elm->next = tcaches_avail; tcaches_avail = elm; malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); @@ -652,8 +733,8 @@ bool tcache_boot(tsdn_t *tsdn) { /* If necessary, clamp opt_lg_tcache_max. */ if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < - SMALL_MAXCLASS) { - tcache_maxclass = SMALL_MAXCLASS; + SC_SMALL_MAXCLASS) { + tcache_maxclass = SC_SMALL_MAXCLASS; } else { tcache_maxclass = (ZU(1) << opt_lg_tcache_max); } @@ -673,7 +754,7 @@ tcache_boot(tsdn_t *tsdn) { } stack_nelms = 0; unsigned i; - for (i = 0; i < NBINS; i++) { + for (i = 0; i < SC_NBINS; i++) { if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_SMALL_MIN; diff --git a/deps/jemalloc/src/hooks.c b/deps/jemalloc/src/test_hooks.c similarity index 79% rename from deps/jemalloc/src/hooks.c rename to deps/jemalloc/src/test_hooks.c index 6266ecd47..ace00d9c4 100644 --- a/deps/jemalloc/src/hooks.c +++ b/deps/jemalloc/src/test_hooks.c @@ -6,7 +6,7 @@ * from outside the generated library, so that we can use them in test code. */ JEMALLOC_EXPORT -void (*hooks_arena_new_hook)() = NULL; +void (*test_hooks_arena_new_hook)() = NULL; JEMALLOC_EXPORT -void (*hooks_libc_hook)() = NULL; +void (*test_hooks_libc_hook)() = NULL; diff --git a/deps/jemalloc/src/tsd.c b/deps/jemalloc/src/tsd.c index c1430682d..a31f6b969 100644 --- a/deps/jemalloc/src/tsd.c +++ b/deps/jemalloc/src/tsd.c @@ -12,12 +12,16 @@ static unsigned ncleanups; static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; +/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP -__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; -__thread bool JEMALLOC_TLS_MODEL tsd_initialized = false; +JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER; +JEMALLOC_TSD_TYPE_ATTR(bool) JEMALLOC_TLS_MODEL tsd_initialized = false; bool tsd_booted = false; #elif (defined(JEMALLOC_TLS)) -__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; +JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER; pthread_key_t tsd_tsd; bool tsd_booted = false; #elif (defined(_WIN32)) @@ -41,6 +45,7 @@ tsd_init_head_t tsd_init_head = { ql_head_initializer(blocks), MALLOC_MUTEX_INITIALIZER }; + tsd_wrapper_t tsd_boot_wrapper = { false, TSD_INITIALIZER @@ -48,17 +53,164 @@ tsd_wrapper_t tsd_boot_wrapper = { bool tsd_booted = false; #endif +JEMALLOC_DIAGNOSTIC_POP /******************************************************************************/ +/* A list of all the tsds in the nominal state. */ +typedef ql_head(tsd_t) tsd_list_t; +static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds); +static malloc_mutex_t tsd_nominal_tsds_lock; + +/* How many slow-path-enabling features are turned on. */ +static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0); + +static bool +tsd_in_nominal_list(tsd_t *tsd) { + tsd_t *tsd_list; + bool found = false; + /* + * We don't know that tsd is nominal; it might not be safe to get data + * out of it here. + */ + malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); + ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + if (tsd == tsd_list) { + found = true; + break; + } + } + malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock); + return found; +} + +static void +tsd_add_nominal(tsd_t *tsd) { + assert(!tsd_in_nominal_list(tsd)); + assert(tsd_state_get(tsd) <= tsd_state_nominal_max); + ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +static void +tsd_remove_nominal(tsd_t *tsd) { + assert(tsd_in_nominal_list(tsd)); + assert(tsd_state_get(tsd) <= tsd_state_nominal_max); + malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +static void +tsd_force_recompute(tsdn_t *tsdn) { + /* + * The stores to tsd->state here need to synchronize with the exchange + * in tsd_slow_update. + */ + atomic_fence(ATOMIC_RELEASE); + malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); + tsd_t *remote_tsd; + ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED) + <= tsd_state_nominal_max); + tsd_atomic_store(&remote_tsd->state, tsd_state_nominal_recompute, + ATOMIC_RELAXED); + } + malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock); +} + +void +tsd_global_slow_inc(tsdn_t *tsdn) { + atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); + /* + * We unconditionally force a recompute, even if the global slow count + * was already positive. If we didn't, then it would be possible for us + * to return to the user, have the user synchronize externally with some + * other thread, and then have that other thread not have picked up the + * update yet (since the original incrementing thread might still be + * making its way through the tsd list). + */ + tsd_force_recompute(tsdn); +} + +void tsd_global_slow_dec(tsdn_t *tsdn) { + atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); + /* See the note in ..._inc(). */ + tsd_force_recompute(tsdn); +} + +static bool +tsd_local_slow(tsd_t *tsd) { + return !tsd_tcache_enabled_get(tsd) + || tsd_reentrancy_level_get(tsd) > 0; +} + +bool +tsd_global_slow() { + return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0; +} + +/******************************************************************************/ + +static uint8_t +tsd_state_compute(tsd_t *tsd) { + if (!tsd_nominal(tsd)) { + return tsd_state_get(tsd); + } + /* We're in *a* nominal state; but which one? */ + if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) { + return tsd_state_nominal_slow; + } else { + return tsd_state_nominal; + } +} + void tsd_slow_update(tsd_t *tsd) { - if (tsd_nominal(tsd)) { - if (malloc_slow || !tsd_tcache_enabled_get(tsd) || - tsd_reentrancy_level_get(tsd) > 0) { - tsd->state = tsd_state_nominal_slow; + uint8_t old_state; + do { + uint8_t new_state = tsd_state_compute(tsd); + old_state = tsd_atomic_exchange(&tsd->state, new_state, + ATOMIC_ACQUIRE); + } while (old_state == tsd_state_nominal_recompute); +} + +void +tsd_state_set(tsd_t *tsd, uint8_t new_state) { + /* Only the tsd module can change the state *to* recompute. */ + assert(new_state != tsd_state_nominal_recompute); + uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED); + if (old_state > tsd_state_nominal_max) { + /* + * Not currently in the nominal list, but it might need to be + * inserted there. + */ + assert(!tsd_in_nominal_list(tsd)); + tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED); + if (new_state <= tsd_state_nominal_max) { + tsd_add_nominal(tsd); + } + } else { + /* + * We're currently nominal. If the new state is non-nominal, + * great; we take ourselves off the list and just enter the new + * state. + */ + assert(tsd_in_nominal_list(tsd)); + if (new_state > tsd_state_nominal_max) { + tsd_remove_nominal(tsd); + tsd_atomic_store(&tsd->state, new_state, + ATOMIC_RELAXED); } else { - tsd->state = tsd_state_nominal; + /* + * This is the tricky case. We're transitioning from + * one nominal state to another. The caller can't know + * about any races that are occuring at the same time, + * so we always have to recompute no matter what. + */ + tsd_slow_update(tsd); } } } @@ -87,6 +239,7 @@ tsd_data_init(tsd_t *tsd) { static void assert_tsd_data_cleanup_done(tsd_t *tsd) { assert(!tsd_nominal(tsd)); + assert(!tsd_in_nominal_list(tsd)); assert(*tsd_arenap_get_unsafe(tsd) == NULL); assert(*tsd_iarenap_get_unsafe(tsd) == NULL); assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true); @@ -97,8 +250,8 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) { static bool tsd_data_init_nocleanup(tsd_t *tsd) { - assert(tsd->state == tsd_state_reincarnated || - tsd->state == tsd_state_minimal_initialized); + assert(tsd_state_get(tsd) == tsd_state_reincarnated || + tsd_state_get(tsd) == tsd_state_minimal_initialized); /* * During reincarnation, there is no guarantee that the cleanup function * will be called (deallocation may happen after all tsd destructors). @@ -117,27 +270,33 @@ tsd_t * tsd_fetch_slow(tsd_t *tsd, bool minimal) { assert(!tsd_fast(tsd)); - if (tsd->state == tsd_state_nominal_slow) { - /* On slow path but no work needed. */ - assert(malloc_slow || !tsd_tcache_enabled_get(tsd) || - tsd_reentrancy_level_get(tsd) > 0 || - *tsd_arenas_tdata_bypassp_get(tsd)); - } else if (tsd->state == tsd_state_uninitialized) { + if (tsd_state_get(tsd) == tsd_state_nominal_slow) { + /* + * On slow path but no work needed. Note that we can't + * necessarily *assert* that we're slow, because we might be + * slow because of an asynchronous modification to global state, + * which might be asynchronously modified *back*. + */ + } else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) { + tsd_slow_update(tsd); + } else if (tsd_state_get(tsd) == tsd_state_uninitialized) { if (!minimal) { - tsd->state = tsd_state_nominal; - tsd_slow_update(tsd); - /* Trigger cleanup handler registration. */ - tsd_set(tsd); - tsd_data_init(tsd); + if (tsd_booted) { + tsd_state_set(tsd, tsd_state_nominal); + tsd_slow_update(tsd); + /* Trigger cleanup handler registration. */ + tsd_set(tsd); + tsd_data_init(tsd); + } } else { - tsd->state = tsd_state_minimal_initialized; + tsd_state_set(tsd, tsd_state_minimal_initialized); tsd_set(tsd); tsd_data_init_nocleanup(tsd); } - } else if (tsd->state == tsd_state_minimal_initialized) { + } else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) { if (!minimal) { /* Switch to fully initialized. */ - tsd->state = tsd_state_nominal; + tsd_state_set(tsd, tsd_state_nominal); assert(*tsd_reentrancy_levelp_get(tsd) >= 1); (*tsd_reentrancy_levelp_get(tsd))--; tsd_slow_update(tsd); @@ -145,12 +304,12 @@ tsd_fetch_slow(tsd_t *tsd, bool minimal) { } else { assert_tsd_data_cleanup_done(tsd); } - } else if (tsd->state == tsd_state_purgatory) { - tsd->state = tsd_state_reincarnated; + } else if (tsd_state_get(tsd) == tsd_state_purgatory) { + tsd_state_set(tsd, tsd_state_reincarnated); tsd_set(tsd); tsd_data_init_nocleanup(tsd); } else { - assert(tsd->state == tsd_state_reincarnated); + assert(tsd_state_get(tsd) == tsd_state_reincarnated); } return tsd; @@ -214,7 +373,7 @@ void tsd_cleanup(void *arg) { tsd_t *tsd = (tsd_t *)arg; - switch (tsd->state) { + switch (tsd_state_get(tsd)) { case tsd_state_uninitialized: /* Do nothing. */ break; @@ -232,7 +391,7 @@ tsd_cleanup(void *arg) { case tsd_state_nominal: case tsd_state_nominal_slow: tsd_do_data_cleanup(tsd); - tsd->state = tsd_state_purgatory; + tsd_state_set(tsd, tsd_state_purgatory); tsd_set(tsd); break; case tsd_state_purgatory: @@ -260,6 +419,10 @@ malloc_tsd_boot0(void) { tsd_t *tsd; ncleanups = 0; + if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock", + WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) { + return NULL; + } if (tsd_boot0()) { return NULL; } @@ -310,7 +473,7 @@ _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { # pragma comment(linker, "/INCLUDE:_tls_callback") # else # pragma comment(linker, "/INCLUDE:_tls_used") -# pragma comment(linker, "/INCLUDE:tls_callback") +# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) ) # endif # pragma section(".CRT$XLY",long,read) #endif @@ -349,3 +512,23 @@ tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) { malloc_mutex_unlock(TSDN_NULL, &head->lock); } #endif + +void +tsd_prefork(tsd_t *tsd) { + malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +void +tsd_postfork_parent(tsd_t *tsd) { + malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +void +tsd_postfork_child(tsd_t *tsd) { + malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_new(&tsd_nominal_tsds); + + if (tsd_state_get(tsd) <= tsd_state_nominal_max) { + tsd_add_nominal(tsd); + } +} diff --git a/deps/jemalloc/test/include/test/jemalloc_test.h.in b/deps/jemalloc/test/include/test/jemalloc_test.h.in index 67caa86bf..c46af5d9b 100644 --- a/deps/jemalloc/test/include/test/jemalloc_test.h.in +++ b/deps/jemalloc/test/include/test/jemalloc_test.h.in @@ -25,7 +25,7 @@ extern "C" { #include "test/jemalloc_test_defs.h" -#ifdef JEMALLOC_OSSPIN +#if defined(JEMALLOC_OSATOMIC) # include #endif @@ -69,7 +69,7 @@ static const bool config_debug = # define JEMALLOC_N(n) @private_namespace@##n # include "jemalloc/internal/private_namespace.h" -# include "jemalloc/internal/hooks.h" +# include "jemalloc/internal/test_hooks.h" /* Hermetic headers. */ # include "jemalloc/internal/assert.h" diff --git a/deps/jemalloc/test/include/test/mtx.h b/deps/jemalloc/test/include/test/mtx.h index 58afbc3d1..066a21379 100644 --- a/deps/jemalloc/test/include/test/mtx.h +++ b/deps/jemalloc/test/include/test/mtx.h @@ -10,8 +10,6 @@ typedef struct { CRITICAL_SECTION lock; #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock lock; -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLock lock; #else pthread_mutex_t lock; #endif diff --git a/deps/jemalloc/test/integration/aligned_alloc.c b/deps/jemalloc/test/integration/aligned_alloc.c index 536b67ea8..4375b172a 100644 --- a/deps/jemalloc/test/integration/aligned_alloc.c +++ b/deps/jemalloc/test/integration/aligned_alloc.c @@ -34,6 +34,17 @@ TEST_BEGIN(test_alignment_errors) { } TEST_END + +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_oom_errors) { size_t alignment, size; void *p; @@ -78,6 +89,9 @@ TEST_BEGIN(test_oom_errors) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + TEST_BEGIN(test_alignment_and_size) { #define NITER 4 size_t alignment, size, total; @@ -124,10 +138,20 @@ TEST_BEGIN(test_alignment_and_size) { } TEST_END +TEST_BEGIN(test_zero_alloc) { + void *res = aligned_alloc(8, 0); + assert(res); + size_t usable = malloc_usable_size(res); + assert(usable > 0); + free(res); +} +TEST_END + int main(void) { return test( test_alignment_errors, test_oom_errors, - test_alignment_and_size); + test_alignment_and_size, + test_zero_alloc); } diff --git a/deps/jemalloc/test/integration/malloc.c b/deps/jemalloc/test/integration/malloc.c new file mode 100644 index 000000000..8b33bc8f3 --- /dev/null +++ b/deps/jemalloc/test/integration/malloc.c @@ -0,0 +1,16 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_zero_alloc) { + void *res = malloc(0); + assert(res); + size_t usable = malloc_usable_size(res); + assert(usable > 0); + free(res); +} +TEST_END + +int +main(void) { + return test( + test_zero_alloc); +} diff --git a/deps/jemalloc/test/integration/mallocx.c b/deps/jemalloc/test/integration/mallocx.c index fd960f30c..645d4db48 100644 --- a/deps/jemalloc/test/integration/mallocx.c +++ b/deps/jemalloc/test/integration/mallocx.c @@ -51,6 +51,16 @@ purge(void) { "Unexpected mallctl error"); } +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_overflow) { size_t largemax; @@ -71,6 +81,38 @@ TEST_BEGIN(test_overflow) { } TEST_END +static void * +remote_alloc(void *arg) { + unsigned arena; + size_t sz = sizeof(unsigned); + assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + size_t large_sz; + sz = sizeof(size_t); + assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large_sz, &sz, + NULL, 0), 0, "Unexpected mallctl failure"); + + void *ptr = mallocx(large_sz, MALLOCX_ARENA(arena) + | MALLOCX_TCACHE_NONE); + void **ret = (void **)arg; + *ret = ptr; + + return NULL; +} + +TEST_BEGIN(test_remote_free) { + thd_t thd; + void *ret; + thd_create(&thd, remote_alloc, (void *)&ret); + thd_join(thd, NULL); + assert_ptr_not_null(ret, "Unexpected mallocx failure"); + + /* Avoid TCACHE_NONE to explicitly test tcache_flush(). */ + dallocx(ret, 0); + mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); +} +TEST_END + TEST_BEGIN(test_oom) { size_t largemax; bool oom; @@ -84,7 +126,7 @@ TEST_BEGIN(test_oom) { largemax = get_large_size(get_nlarge()-1); oom = false; for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { - ptrs[i] = mallocx(largemax, 0); + ptrs[i] = mallocx(largemax, MALLOCX_ARENA(0)); if (ptrs[i] == NULL) { oom = true; } @@ -113,6 +155,9 @@ TEST_BEGIN(test_oom) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + TEST_BEGIN(test_basic) { #define MAXSZ (((size_t)1) << 23) size_t sz; @@ -178,12 +223,12 @@ TEST_BEGIN(test_alignment_and_size) { sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { for (i = 0; i < NITER; i++) { nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | - MALLOCX_ZERO); + MALLOCX_ZERO | MALLOCX_ARENA(0)); assert_zu_ne(nsz, 0, "nallocx() error for alignment=%zu, " "size=%zu (%#zx)", alignment, sz, sz); ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) | - MALLOCX_ZERO); + MALLOCX_ZERO | MALLOCX_ARENA(0)); assert_ptr_not_null(ps[i], "mallocx() error for alignment=%zu, " "size=%zu (%#zx)", alignment, sz, sz); @@ -223,6 +268,7 @@ main(void) { return test( test_overflow, test_oom, + test_remote_free, test_basic, test_alignment_and_size); } diff --git a/deps/jemalloc/test/integration/overflow.c b/deps/jemalloc/test/integration/overflow.c index 6a9785b2e..748ebb677 100644 --- a/deps/jemalloc/test/integration/overflow.c +++ b/deps/jemalloc/test/integration/overflow.c @@ -1,5 +1,15 @@ #include "test/jemalloc_test.h" +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_overflow) { unsigned nlextents; size_t mib[4]; @@ -39,6 +49,9 @@ TEST_BEGIN(test_overflow) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + int main(void) { return test( diff --git a/deps/jemalloc/test/integration/posix_memalign.c b/deps/jemalloc/test/integration/posix_memalign.c index 2c2726de8..d992260a2 100644 --- a/deps/jemalloc/test/integration/posix_memalign.c +++ b/deps/jemalloc/test/integration/posix_memalign.c @@ -85,9 +85,10 @@ TEST_BEGIN(test_alignment_and_size) { alignment <= MAXALIGN; alignment <<= 1) { total = 0; - for (size = 1; + for (size = 0; size < 3 * alignment && size < (1U << 31); - size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + size += ((size == 0) ? 1 : + (alignment >> (LG_SIZEOF_PTR-1)) - 1)) { for (i = 0; i < NITER; i++) { err = posix_memalign(&ps[i], alignment, size); diff --git a/deps/jemalloc/test/integration/rallocx.c b/deps/jemalloc/test/integration/rallocx.c index 7821ca5f5..08ed08d3f 100644 --- a/deps/jemalloc/test/integration/rallocx.c +++ b/deps/jemalloc/test/integration/rallocx.c @@ -208,6 +208,16 @@ TEST_BEGIN(test_lg_align_and_zero) { } TEST_END +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_overflow) { size_t largemax; void *p; @@ -234,6 +244,9 @@ TEST_BEGIN(test_overflow) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + int main(void) { return test( diff --git a/deps/jemalloc/test/integration/slab_sizes.c b/deps/jemalloc/test/integration/slab_sizes.c new file mode 100644 index 000000000..af250c3f4 --- /dev/null +++ b/deps/jemalloc/test/integration/slab_sizes.c @@ -0,0 +1,80 @@ +#include "test/jemalloc_test.h" + +/* Note that this test relies on the unusual slab sizes set in slab_sizes.sh. */ + +TEST_BEGIN(test_slab_sizes) { + unsigned nbins; + size_t page; + size_t sizemib[4]; + size_t slabmib[4]; + size_t len; + + len = sizeof(nbins); + assert_d_eq(mallctl("arenas.nbins", &nbins, &len, NULL, 0), 0, + "nbins mallctl failure"); + + len = sizeof(page); + assert_d_eq(mallctl("arenas.page", &page, &len, NULL, 0), 0, + "page mallctl failure"); + + len = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.size", sizemib, &len), 0, + "bin size mallctlnametomib failure"); + + len = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.slab_size", slabmib, &len), + 0, "slab size mallctlnametomib failure"); + + size_t biggest_slab_seen = 0; + + for (unsigned i = 0; i < nbins; i++) { + size_t bin_size; + size_t slab_size; + len = sizeof(size_t); + sizemib[2] = i; + slabmib[2] = i; + assert_d_eq(mallctlbymib(sizemib, 4, (void *)&bin_size, &len, + NULL, 0), 0, "bin size mallctlbymib failure"); + + len = sizeof(size_t); + assert_d_eq(mallctlbymib(slabmib, 4, (void *)&slab_size, &len, + NULL, 0), 0, "slab size mallctlbymib failure"); + + if (bin_size < 100) { + /* + * Then we should be as close to 17 as possible. Since + * not all page sizes are valid (because of bitmap + * limitations on the number of items in a slab), we + * should at least make sure that the number of pages + * goes up. + */ + assert_zu_ge(slab_size, biggest_slab_seen, + "Slab sizes should go up"); + biggest_slab_seen = slab_size; + } else if ( + (100 <= bin_size && bin_size < 128) + || (128 < bin_size && bin_size <= 200)) { + assert_zu_eq(slab_size, page, + "Forced-small slabs should be small"); + } else if (bin_size == 128) { + assert_zu_eq(slab_size, 2 * page, + "Forced-2-page slab should be 2 pages"); + } else if (200 < bin_size && bin_size <= 4096) { + assert_zu_ge(slab_size, biggest_slab_seen, + "Slab sizes should go up"); + biggest_slab_seen = slab_size; + } + } + /* + * For any reasonable configuration, 17 pages should be a valid slab + * size for 4096-byte items. + */ + assert_zu_eq(biggest_slab_seen, 17 * page, "Didn't hit page target"); +} +TEST_END + +int +main(void) { + return test( + test_slab_sizes); +} diff --git a/deps/jemalloc/test/integration/slab_sizes.sh b/deps/jemalloc/test/integration/slab_sizes.sh new file mode 100644 index 000000000..07e3db81b --- /dev/null +++ b/deps/jemalloc/test/integration/slab_sizes.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# Some screwy-looking slab sizes. +export MALLOC_CONF="slab_sizes:1-4096:17|100-200:1|128-128:2" diff --git a/deps/jemalloc/test/integration/smallocx.c b/deps/jemalloc/test/integration/smallocx.c new file mode 100644 index 000000000..2486752be --- /dev/null +++ b/deps/jemalloc/test/integration/smallocx.c @@ -0,0 +1,312 @@ +#include "test/jemalloc_test.h" +#include "jemalloc/jemalloc_macros.h" + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +#ifndef JEMALLOC_VERSION_GID_IDENT + #error "JEMALLOC_VERSION_GID_IDENT not defined" +#endif + +#define JOIN(x, y) x ## y +#define JOIN2(x, y) JOIN(x, y) +#define smallocx JOIN2(smallocx_, JEMALLOC_VERSION_GID_IDENT) + +typedef struct { + void *ptr; + size_t size; +} smallocx_return_t; + +extern smallocx_return_t +smallocx(size_t size, int flags); + +static unsigned +get_nsizes_impl(const char *cmd) { + unsigned ret; + size_t z; + + z = sizeof(unsigned); + assert_d_eq(mallctl(cmd, (void *)&ret, &z, NULL, 0), 0, + "Unexpected mallctl(\"%s\", ...) failure", cmd); + + return ret; +} + +static unsigned +get_nlarge(void) { + return get_nsizes_impl("arenas.nlextents"); +} + +static size_t +get_size_impl(const char *cmd, size_t ind) { + size_t ret; + size_t z; + size_t mib[4]; + size_t miblen = 4; + + z = sizeof(size_t); + assert_d_eq(mallctlnametomib(cmd, mib, &miblen), + 0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd); + mib[2] = ind; + z = sizeof(size_t); + assert_d_eq(mallctlbymib(mib, miblen, (void *)&ret, &z, NULL, 0), + 0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind); + + return ret; +} + +static size_t +get_large_size(size_t ind) { + return get_size_impl("arenas.lextent.0.size", ind); +} + +/* + * On systems which can't merge extents, tests that call this function generate + * a lot of dirty memory very quickly. Purging between cycles mitigates + * potential OOM on e.g. 32-bit Windows. + */ +static void +purge(void) { + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl error"); +} + +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + +TEST_BEGIN(test_overflow) { + size_t largemax; + + largemax = get_large_size(get_nlarge()-1); + + assert_ptr_null(smallocx(largemax+1, 0).ptr, + "Expected OOM for smallocx(size=%#zx, 0)", largemax+1); + + assert_ptr_null(smallocx(ZU(PTRDIFF_MAX)+1, 0).ptr, + "Expected OOM for smallocx(size=%#zx, 0)", ZU(PTRDIFF_MAX)+1); + + assert_ptr_null(smallocx(SIZE_T_MAX, 0).ptr, + "Expected OOM for smallocx(size=%#zx, 0)", SIZE_T_MAX); + + assert_ptr_null(smallocx(1, MALLOCX_ALIGN(ZU(PTRDIFF_MAX)+1)).ptr, + "Expected OOM for smallocx(size=1, MALLOCX_ALIGN(%#zx))", + ZU(PTRDIFF_MAX)+1); +} +TEST_END + +static void * +remote_alloc(void *arg) { + unsigned arena; + size_t sz = sizeof(unsigned); + assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + size_t large_sz; + sz = sizeof(size_t); + assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large_sz, &sz, + NULL, 0), 0, "Unexpected mallctl failure"); + + smallocx_return_t r + = smallocx(large_sz, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE); + void *ptr = r.ptr; + assert_zu_eq(r.size, + nallocx(large_sz, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE), + "Expected smalloc(size,flags).size == nallocx(size,flags)"); + void **ret = (void **)arg; + *ret = ptr; + + return NULL; +} + +TEST_BEGIN(test_remote_free) { + thd_t thd; + void *ret; + thd_create(&thd, remote_alloc, (void *)&ret); + thd_join(thd, NULL); + assert_ptr_not_null(ret, "Unexpected smallocx failure"); + + /* Avoid TCACHE_NONE to explicitly test tcache_flush(). */ + dallocx(ret, 0); + mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); +} +TEST_END + +TEST_BEGIN(test_oom) { + size_t largemax; + bool oom; + void *ptrs[3]; + unsigned i; + + /* + * It should be impossible to allocate three objects that each consume + * nearly half the virtual address space. + */ + largemax = get_large_size(get_nlarge()-1); + oom = false; + for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { + ptrs[i] = smallocx(largemax, 0).ptr; + if (ptrs[i] == NULL) { + oom = true; + } + } + assert_true(oom, + "Expected OOM during series of calls to smallocx(size=%zu, 0)", + largemax); + for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { + if (ptrs[i] != NULL) { + dallocx(ptrs[i], 0); + } + } + purge(); + +#if LG_SIZEOF_PTR == 3 + assert_ptr_null(smallocx(0x8000000000000000ULL, + MALLOCX_ALIGN(0x8000000000000000ULL)).ptr, + "Expected OOM for smallocx()"); + assert_ptr_null(smallocx(0x8000000000000000ULL, + MALLOCX_ALIGN(0x80000000)).ptr, + "Expected OOM for smallocx()"); +#else + assert_ptr_null(smallocx(0x80000000UL, MALLOCX_ALIGN(0x80000000UL)).ptr, + "Expected OOM for smallocx()"); +#endif +} +TEST_END + +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + +TEST_BEGIN(test_basic) { +#define MAXSZ (((size_t)1) << 23) + size_t sz; + + for (sz = 1; sz < MAXSZ; sz = nallocx(sz, 0) + 1) { + smallocx_return_t ret; + size_t nsz, rsz, smz; + void *p; + nsz = nallocx(sz, 0); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + ret = smallocx(sz, 0); + p = ret.ptr; + smz = ret.size; + assert_ptr_not_null(p, + "Unexpected smallocx(size=%zx, flags=0) error", sz); + rsz = sallocx(p, 0); + assert_zu_ge(rsz, sz, "Real size smaller than expected"); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch"); + assert_zu_eq(nsz, smz, "nallocx()/smallocx() size mismatch"); + dallocx(p, 0); + + ret = smallocx(sz, 0); + p = ret.ptr; + smz = ret.size; + assert_ptr_not_null(p, + "Unexpected smallocx(size=%zx, flags=0) error", sz); + dallocx(p, 0); + + nsz = nallocx(sz, MALLOCX_ZERO); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + assert_zu_ne(smz, 0, "Unexpected smallocx() error"); + ret = smallocx(sz, MALLOCX_ZERO); + p = ret.ptr; + assert_ptr_not_null(p, + "Unexpected smallocx(size=%zx, flags=MALLOCX_ZERO) error", + nsz); + rsz = sallocx(p, 0); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch"); + assert_zu_eq(nsz, smz, "nallocx()/smallocx() size mismatch"); + dallocx(p, 0); + purge(); + } +#undef MAXSZ +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) { + const char *percpu_arena; + size_t sz = sizeof(percpu_arena); + + if(mallctl("opt.percpu_arena", (void *)&percpu_arena, &sz, NULL, 0) || + strcmp(percpu_arena, "disabled") != 0) { + test_skip("test_alignment_and_size skipped: " + "not working with percpu arena."); + }; +#define MAXALIGN (((size_t)1) << 23) +#define NITER 4 + size_t nsz, rsz, smz, alignment, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) { + ps[i] = NULL; + } + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + assert_zu_ne(nsz, 0, + "nallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + smallocx_return_t ret + = smallocx(sz, MALLOCX_ALIGN(alignment) | MALLOCX_ZERO); + ps[i] = ret.ptr; + assert_ptr_not_null(ps[i], + "smallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + rsz = sallocx(ps[i], 0); + smz = ret.size; + assert_zu_ge(rsz, sz, + "Real size smaller than expected for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, rsz, + "nallocx()/sallocx() size mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, smz, + "nallocx()/smallocx() size mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_ptr_null( + (void *)((uintptr_t)ps[i] & (alignment-1)), + "%p inadequately aligned for" + " alignment=%zu, size=%zu", ps[i], + alignment, sz); + total += rsz; + if (total >= (MAXALIGN << 1)) { + break; + } + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + dallocx(ps[i], 0); + ps[i] = NULL; + } + } + } + purge(); + } +#undef MAXALIGN +#undef NITER +} +TEST_END + +int +main(void) { + return test( + test_overflow, + test_oom, + test_remote_free, + test_basic, + test_alignment_and_size); +} diff --git a/deps/jemalloc/test/integration/smallocx.sh b/deps/jemalloc/test/integration/smallocx.sh new file mode 100644 index 000000000..d07f10f3c --- /dev/null +++ b/deps/jemalloc/test/integration/smallocx.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="junk:false" +fi diff --git a/deps/jemalloc/test/src/mtx.c b/deps/jemalloc/test/src/mtx.c index a393c01fc..d9ce375c3 100644 --- a/deps/jemalloc/test/src/mtx.c +++ b/deps/jemalloc/test/src/mtx.c @@ -13,8 +13,6 @@ mtx_init(mtx_t *mtx) { } #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) mtx->lock = OS_UNFAIR_LOCK_INIT; -#elif (defined(JEMALLOC_OSSPIN)) - mtx->lock = 0; #else pthread_mutexattr_t attr; @@ -35,7 +33,6 @@ void mtx_fini(mtx_t *mtx) { #ifdef _WIN32 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -#elif (defined(JEMALLOC_OSSPIN)) #else pthread_mutex_destroy(&mtx->lock); #endif @@ -47,8 +44,6 @@ mtx_lock(mtx_t *mtx) { EnterCriticalSection(&mtx->lock); #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock_lock(&mtx->lock); -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLockLock(&mtx->lock); #else pthread_mutex_lock(&mtx->lock); #endif @@ -60,8 +55,6 @@ mtx_unlock(mtx_t *mtx) { LeaveCriticalSection(&mtx->lock); #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock_unlock(&mtx->lock); -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLockUnlock(&mtx->lock); #else pthread_mutex_unlock(&mtx->lock); #endif diff --git a/deps/jemalloc/test/src/test.c b/deps/jemalloc/test/src/test.c index 01a4d7380..f97ce4d18 100644 --- a/deps/jemalloc/test/src/test.c +++ b/deps/jemalloc/test/src/test.c @@ -48,12 +48,12 @@ do_hook(bool *hook_ran, void (**hook)()) { static void libc_reentrancy_hook() { - do_hook(&libc_hook_ran, &hooks_libc_hook); + do_hook(&libc_hook_ran, &test_hooks_libc_hook); } static void arena_new_reentrancy_hook() { - do_hook(&arena_new_hook_ran, &hooks_arena_new_hook); + do_hook(&arena_new_hook_ran, &test_hooks_arena_new_hook); } /* Actual test infrastructure. */ @@ -110,6 +110,20 @@ p_test_fini(void) { test_status_string(test_status)); } +static void +check_global_slow(test_status_t *status) { +#ifdef JEMALLOC_UNIT_TEST + /* + * This check needs to peek into tsd internals, which is why it's only + * exposed in unit tests. + */ + if (tsd_global_slow()) { + malloc_printf("Testing increased global slow count\n"); + *status = test_status_fail; + } +#endif +} + static test_status_t p_test_impl(bool do_malloc_init, bool do_reentrant, test_t *t, va_list ap) { test_status_t ret; @@ -131,28 +145,31 @@ p_test_impl(bool do_malloc_init, bool do_reentrant, test_t *t, va_list ap) { for (; t != NULL; t = va_arg(ap, test_t *)) { /* Non-reentrant run. */ reentrancy = non_reentrant; - hooks_arena_new_hook = hooks_libc_hook = NULL; + test_hooks_arena_new_hook = test_hooks_libc_hook = NULL; t(); if (test_status > ret) { ret = test_status; } + check_global_slow(&ret); /* Reentrant run. */ if (do_reentrant) { reentrancy = libc_reentrant; - hooks_arena_new_hook = NULL; - hooks_libc_hook = &libc_reentrancy_hook; + test_hooks_arena_new_hook = NULL; + test_hooks_libc_hook = &libc_reentrancy_hook; t(); if (test_status > ret) { ret = test_status; } + check_global_slow(&ret); reentrancy = arena_new_reentrant; - hooks_libc_hook = NULL; - hooks_arena_new_hook = &arena_new_reentrancy_hook; + test_hooks_libc_hook = NULL; + test_hooks_arena_new_hook = &arena_new_reentrancy_hook; t(); if (test_status > ret) { ret = test_status; } + check_global_slow(&ret); } } diff --git a/deps/jemalloc/test/stress/hookbench.c b/deps/jemalloc/test/stress/hookbench.c new file mode 100644 index 000000000..97e90b0e3 --- /dev/null +++ b/deps/jemalloc/test/stress/hookbench.c @@ -0,0 +1,73 @@ +#include "test/jemalloc_test.h" + +static void +noop_alloc_hook(void *extra, hook_alloc_t type, void *result, + uintptr_t result_raw, uintptr_t args_raw[3]) { +} + +static void +noop_dalloc_hook(void *extra, hook_dalloc_t type, void *address, + uintptr_t args_raw[3]) { +} + +static void +noop_expand_hook(void *extra, hook_expand_t type, void *address, + size_t old_usize, size_t new_usize, uintptr_t result_raw, + uintptr_t args_raw[4]) { +} + +static void +malloc_free_loop(int iters) { + for (int i = 0; i < iters; i++) { + void *p = mallocx(1, 0); + free(p); + } +} + +static void +test_hooked(int iters) { + hooks_t hooks = {&noop_alloc_hook, &noop_dalloc_hook, &noop_expand_hook, + NULL}; + + int err; + void *handles[HOOK_MAX]; + size_t sz = sizeof(handles[0]); + + for (int i = 0; i < HOOK_MAX; i++) { + err = mallctl("experimental.hooks.install", &handles[i], + &sz, &hooks, sizeof(hooks)); + assert(err == 0); + + timedelta_t timer; + timer_start(&timer); + malloc_free_loop(iters); + timer_stop(&timer); + malloc_printf("With %d hook%s: %"FMTu64"us\n", i + 1, + i + 1 == 1 ? "" : "s", timer_usec(&timer)); + } + for (int i = 0; i < HOOK_MAX; i++) { + err = mallctl("experimental.hooks.remove", NULL, NULL, + &handles[i], sizeof(handles[i])); + assert(err == 0); + } +} + +static void +test_unhooked(int iters) { + timedelta_t timer; + timer_start(&timer); + malloc_free_loop(iters); + timer_stop(&timer); + + malloc_printf("Without hooks: %"FMTu64"us\n", timer_usec(&timer)); +} + +int +main(void) { + /* Initialize */ + free(mallocx(1, 0)); + int iters = 10 * 1000 * 1000; + malloc_printf("Benchmarking hooks with %d iterations:\n", iters); + test_hooked(iters); + test_unhooked(iters); +} diff --git a/deps/jemalloc/test/unit/arena_reset.c b/deps/jemalloc/test/unit/arena_reset.c index f5fb24d1e..b182f31a6 100644 --- a/deps/jemalloc/test/unit/arena_reset.c +++ b/deps/jemalloc/test/unit/arena_reset.c @@ -77,7 +77,7 @@ vsalloc(tsdn_t *tsdn, const void *ptr) { return 0; } - if (szind == NSIZES) { + if (szind == SC_NSIZES) { return 0; } @@ -142,7 +142,7 @@ do_arena_reset_post(void **ptrs, unsigned nptrs, unsigned arena_ind) { if (have_background_thread) { malloc_mutex_lock(tsdn, - &background_thread_info[arena_ind % ncpus].mtx); + &background_thread_info_get(arena_ind)->mtx); } /* Verify allocations no longer exist. */ for (i = 0; i < nptrs; i++) { @@ -151,7 +151,7 @@ do_arena_reset_post(void **ptrs, unsigned nptrs, unsigned arena_ind) { } if (have_background_thread) { malloc_mutex_unlock(tsdn, - &background_thread_info[arena_ind % ncpus].mtx); + &background_thread_info_get(arena_ind)->mtx); } free(ptrs); @@ -279,8 +279,11 @@ extent_dalloc_unmap(extent_hooks_t *extent_hooks, void *addr, size_t size, if (!try_dalloc) { return true; } - pages_unmap(addr, size); did_dalloc = true; + if (!maps_coalesce && opt_retain) { + return true; + } + pages_unmap(addr, size); return false; } @@ -304,7 +307,9 @@ TEST_BEGIN(test_arena_destroy_hooks_unmap) { unsigned nptrs; extent_hooks_prep(); - try_decommit = false; + if (maps_coalesce) { + try_decommit = false; + } memcpy(&hooks_orig, &hooks, sizeof(extent_hooks_t)); memcpy(&hooks, &hooks_unmap, sizeof(extent_hooks_t)); diff --git a/deps/jemalloc/test/unit/background_thread_enable.c b/deps/jemalloc/test/unit/background_thread_enable.c index ff95e672c..d894e9371 100644 --- a/deps/jemalloc/test/unit/background_thread_enable.c +++ b/deps/jemalloc/test/unit/background_thread_enable.c @@ -33,20 +33,19 @@ TEST_END TEST_BEGIN(test_max_background_threads) { test_skip_if(!have_background_thread); - size_t maxt; - size_t opt_maxt; - size_t sz_m = sizeof(maxt); + size_t max_n_thds; + size_t opt_max_n_thds; + size_t sz_m = sizeof(max_n_thds); assert_d_eq(mallctl("opt.max_background_threads", - &opt_maxt, &sz_m, NULL, 0), 0, - "Failed to get opt.max_background_threads"); - assert_d_eq(mallctl("max_background_threads", &maxt, &sz_m, NULL, 0), 0, - "Failed to get max background threads"); - assert_zu_eq(20, maxt, "should be ncpus"); - assert_zu_eq(opt_maxt, maxt, - "max_background_threads and " - "opt.max_background_threads should match"); - assert_d_eq(mallctl("max_background_threads", NULL, NULL, &maxt, sz_m), - 0, "Failed to set max background threads"); + &opt_max_n_thds, &sz_m, NULL, 0), 0, + "Failed to get opt.max_background_threads"); + assert_d_eq(mallctl("max_background_threads", &max_n_thds, &sz_m, NULL, + 0), 0, "Failed to get max background threads"); + assert_zu_eq(opt_max_n_thds, max_n_thds, + "max_background_threads and " + "opt.max_background_threads should match"); + assert_d_eq(mallctl("max_background_threads", NULL, NULL, &max_n_thds, + sz_m), 0, "Failed to set max background threads"); unsigned id; size_t sz_u = sizeof(unsigned); @@ -60,18 +59,21 @@ TEST_BEGIN(test_max_background_threads) { size_t sz_b = sizeof(bool); assert_d_eq(mallctl("background_thread", NULL, NULL, &enable, sz_b), 0, "Failed to enable background threads"); - assert_zu_eq(n_background_threads, maxt, - "Number of background threads should be 3.\n"); - maxt = 10; - assert_d_eq(mallctl("max_background_threads", NULL, NULL, &maxt, sz_m), - 0, "Failed to set max background threads"); - assert_zu_eq(n_background_threads, maxt, - "Number of background threads should be 10.\n"); - maxt = 3; - assert_d_eq(mallctl("max_background_threads", NULL, NULL, &maxt, sz_m), - 0, "Failed to set max background threads"); - assert_zu_eq(n_background_threads, maxt, - "Number of background threads should be 3.\n"); + assert_zu_eq(n_background_threads, max_n_thds, + "Number of background threads should not change.\n"); + size_t new_max_thds = max_n_thds - 1; + if (new_max_thds > 0) { + assert_d_eq(mallctl("max_background_threads", NULL, NULL, + &new_max_thds, sz_m), 0, + "Failed to set max background threads"); + assert_zu_eq(n_background_threads, new_max_thds, + "Number of background threads should decrease by 1.\n"); + } + new_max_thds = 1; + assert_d_eq(mallctl("max_background_threads", NULL, NULL, &new_max_thds, + sz_m), 0, "Failed to set max background threads"); + assert_zu_eq(n_background_threads, new_max_thds, + "Number of background threads should be 1.\n"); } TEST_END diff --git a/deps/jemalloc/test/unit/binshard.c b/deps/jemalloc/test/unit/binshard.c new file mode 100644 index 000000000..d7a8df8fc --- /dev/null +++ b/deps/jemalloc/test/unit/binshard.c @@ -0,0 +1,154 @@ +#include "test/jemalloc_test.h" + +/* Config -- "narenas:1,bin_shards:1-160:16|129-512:4|256-256:8" */ + +#define NTHREADS 16 +#define REMOTE_NALLOC 256 + +static void * +thd_producer(void *varg) { + void **mem = varg; + unsigned arena, i; + size_t sz; + + sz = sizeof(arena); + /* Remote arena. */ + assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + for (i = 0; i < REMOTE_NALLOC / 2; i++) { + mem[i] = mallocx(1, MALLOCX_TCACHE_NONE | MALLOCX_ARENA(arena)); + } + + /* Remote bin. */ + for (; i < REMOTE_NALLOC; i++) { + mem[i] = mallocx(1, MALLOCX_TCACHE_NONE | MALLOCX_ARENA(0)); + } + + return NULL; +} + +TEST_BEGIN(test_producer_consumer) { + thd_t thds[NTHREADS]; + void *mem[NTHREADS][REMOTE_NALLOC]; + unsigned i; + + /* Create producer threads to allocate. */ + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_producer, mem[i]); + } + for (i = 0; i < NTHREADS; i++) { + thd_join(thds[i], NULL); + } + /* Remote deallocation by the current thread. */ + for (i = 0; i < NTHREADS; i++) { + for (unsigned j = 0; j < REMOTE_NALLOC; j++) { + assert_ptr_not_null(mem[i][j], + "Unexpected remote allocation failure"); + dallocx(mem[i][j], 0); + } + } +} +TEST_END + +static void * +thd_start(void *varg) { + void *ptr, *ptr2; + extent_t *extent; + unsigned shard1, shard2; + + tsdn_t *tsdn = tsdn_fetch(); + /* Try triggering allocations from sharded bins. */ + for (unsigned i = 0; i < 1024; i++) { + ptr = mallocx(1, MALLOCX_TCACHE_NONE); + ptr2 = mallocx(129, MALLOCX_TCACHE_NONE); + + extent = iealloc(tsdn, ptr); + shard1 = extent_binshard_get(extent); + dallocx(ptr, 0); + assert_u_lt(shard1, 16, "Unexpected bin shard used"); + + extent = iealloc(tsdn, ptr2); + shard2 = extent_binshard_get(extent); + dallocx(ptr2, 0); + assert_u_lt(shard2, 4, "Unexpected bin shard used"); + + if (shard1 > 0 || shard2 > 0) { + /* Triggered sharded bin usage. */ + return (void *)(uintptr_t)shard1; + } + } + + return NULL; +} + +TEST_BEGIN(test_bin_shard_mt) { + test_skip_if(have_percpu_arena && + PERCPU_ARENA_ENABLED(opt_percpu_arena)); + + thd_t thds[NTHREADS]; + unsigned i; + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_start, NULL); + } + bool sharded = false; + for (i = 0; i < NTHREADS; i++) { + void *ret; + thd_join(thds[i], &ret); + if (ret != NULL) { + sharded = true; + } + } + assert_b_eq(sharded, true, "Did not find sharded bins"); +} +TEST_END + +TEST_BEGIN(test_bin_shard) { + unsigned nbins, i; + size_t mib[4], mib2[4]; + size_t miblen, miblen2, len; + + len = sizeof(nbins); + assert_d_eq(mallctl("arenas.nbins", (void *)&nbins, &len, NULL, 0), 0, + "Unexpected mallctl() failure"); + + miblen = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.nshards", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + miblen2 = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.size", mib2, &miblen2), 0, + "Unexpected mallctlnametomib() failure"); + + for (i = 0; i < nbins; i++) { + uint32_t nshards; + size_t size, sz1, sz2; + + mib[2] = i; + sz1 = sizeof(nshards); + assert_d_eq(mallctlbymib(mib, miblen, (void *)&nshards, &sz1, + NULL, 0), 0, "Unexpected mallctlbymib() failure"); + + mib2[2] = i; + sz2 = sizeof(size); + assert_d_eq(mallctlbymib(mib2, miblen2, (void *)&size, &sz2, + NULL, 0), 0, "Unexpected mallctlbymib() failure"); + + if (size >= 1 && size <= 128) { + assert_u_eq(nshards, 16, "Unexpected nshards"); + } else if (size == 256) { + assert_u_eq(nshards, 8, "Unexpected nshards"); + } else if (size > 128 && size <= 512) { + assert_u_eq(nshards, 4, "Unexpected nshards"); + } else { + assert_u_eq(nshards, 1, "Unexpected nshards"); + } + } +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_bin_shard, + test_bin_shard_mt, + test_producer_consumer); +} diff --git a/deps/jemalloc/test/unit/binshard.sh b/deps/jemalloc/test/unit/binshard.sh new file mode 100644 index 000000000..c1d58c883 --- /dev/null +++ b/deps/jemalloc/test/unit/binshard.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export MALLOC_CONF="narenas:1,bin_shards:1-160:16|129-512:4|256-256:8" diff --git a/deps/jemalloc/test/unit/bit_util.c b/deps/jemalloc/test/unit/bit_util.c index 42a97013d..b747deb43 100644 --- a/deps/jemalloc/test/unit/bit_util.c +++ b/deps/jemalloc/test/unit/bit_util.c @@ -48,10 +48,64 @@ TEST_BEGIN(test_pow2_ceil_zu) { } TEST_END +void +assert_lg_ceil_range(size_t input, unsigned answer) { + if (input == 1) { + assert_u_eq(0, answer, "Got %u as lg_ceil of 1", answer); + return; + } + assert_zu_le(input, (ZU(1) << answer), + "Got %u as lg_ceil of %zu", answer, input); + assert_zu_gt(input, (ZU(1) << (answer - 1)), + "Got %u as lg_ceil of %zu", answer, input); +} + +void +assert_lg_floor_range(size_t input, unsigned answer) { + if (input == 1) { + assert_u_eq(0, answer, "Got %u as lg_floor of 1", answer); + return; + } + assert_zu_ge(input, (ZU(1) << answer), + "Got %u as lg_floor of %zu", answer, input); + assert_zu_lt(input, (ZU(1) << (answer + 1)), + "Got %u as lg_floor of %zu", answer, input); +} + +TEST_BEGIN(test_lg_ceil_floor) { + for (size_t i = 1; i < 10 * 1000 * 1000; i++) { + assert_lg_ceil_range(i, lg_ceil(i)); + assert_lg_ceil_range(i, LG_CEIL(i)); + assert_lg_floor_range(i, lg_floor(i)); + assert_lg_floor_range(i, LG_FLOOR(i)); + } + for (int i = 10; i < 8 * (1 << LG_SIZEOF_PTR) - 5; i++) { + for (size_t j = 0; j < (1 << 4); j++) { + size_t num1 = ((size_t)1 << i) + - j * ((size_t)1 << (i - 4)); + size_t num2 = ((size_t)1 << i) + + j * ((size_t)1 << (i - 4)); + assert_zu_ne(num1, 0, "Invalid lg argument"); + assert_zu_ne(num2, 0, "Invalid lg argument"); + assert_lg_ceil_range(num1, lg_ceil(num1)); + assert_lg_ceil_range(num1, LG_CEIL(num1)); + assert_lg_ceil_range(num2, lg_ceil(num2)); + assert_lg_ceil_range(num2, LG_CEIL(num2)); + + assert_lg_floor_range(num1, lg_floor(num1)); + assert_lg_floor_range(num1, LG_FLOOR(num1)); + assert_lg_floor_range(num2, lg_floor(num2)); + assert_lg_floor_range(num2, LG_FLOOR(num2)); + } + } +} +TEST_END + int main(void) { return test( test_pow2_ceil_u64, test_pow2_ceil_u32, - test_pow2_ceil_zu); + test_pow2_ceil_zu, + test_lg_ceil_floor); } diff --git a/deps/jemalloc/test/unit/decay.c b/deps/jemalloc/test/unit/decay.c index f727bf931..cf3c07960 100644 --- a/deps/jemalloc/test/unit/decay.c +++ b/deps/jemalloc/test/unit/decay.c @@ -121,6 +121,12 @@ get_arena_dirty_npurge(unsigned arena_ind) { return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind); } +static uint64_t +get_arena_dirty_purged(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind); +} + static uint64_t get_arena_muzzy_npurge(unsigned arena_ind) { do_epoch(); @@ -559,7 +565,7 @@ TEST_BEGIN(test_decay_now) { TEST_END TEST_BEGIN(test_decay_never) { - test_skip_if(check_background_thread_enabled()); + test_skip_if(check_background_thread_enabled() || !config_stats); unsigned arena_ind = do_arena_create(-1, -1); int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; @@ -579,8 +585,8 @@ TEST_BEGIN(test_decay_never) { dallocx(ptrs[i], flags); size_t pdirty = get_arena_pdirty(arena_ind); size_t pmuzzy = get_arena_pmuzzy(arena_ind); - assert_zu_gt(pdirty, pdirty_prev, - "Expected dirty pages to increase."); + assert_zu_gt(pdirty + (size_t)get_arena_dirty_purged(arena_ind), + pdirty_prev, "Expected dirty pages to increase."); assert_zu_eq(pmuzzy, 0, "Unexpected muzzy pages"); pdirty_prev = pdirty; } diff --git a/deps/jemalloc/test/unit/emitter.c b/deps/jemalloc/test/unit/emitter.c index 535c7cf1d..b4a693f4b 100644 --- a/deps/jemalloc/test/unit/emitter.c +++ b/deps/jemalloc/test/unit/emitter.c @@ -169,7 +169,7 @@ static void emit_nested_dict(emitter_t *emitter) { emitter_end(emitter); } -static const char *nested_dict_json = +static const char *nested_object_json = "{\n" "\t\"json1\": {\n" "\t\t\"json2\": {\n" @@ -183,7 +183,7 @@ static const char *nested_dict_json = "\t}\n" "}\n"; -static const char *nested_dict_table = +static const char *nested_object_table = "Dict 1\n" " Dict 2\n" " A primitive: 123\n" @@ -192,8 +192,8 @@ static const char *nested_dict_table = " Another primitive: 123\n"; TEST_BEGIN(test_nested_dict) { - assert_emit_output(&emit_nested_dict, nested_dict_json, - nested_dict_table); + assert_emit_output(&emit_nested_dict, nested_object_json, + nested_object_table); } TEST_END @@ -256,13 +256,14 @@ emit_modal(emitter_t *emitter) { int val = 123; emitter_begin(emitter); emitter_dict_begin(emitter, "j0", "T0"); - emitter_json_dict_begin(emitter, "j1"); + emitter_json_key(emitter, "j1"); + emitter_json_object_begin(emitter); emitter_kv(emitter, "i1", "I1", emitter_type_int, &val); emitter_json_kv(emitter, "i2", emitter_type_int, &val); emitter_table_kv(emitter, "I3", emitter_type_int, &val); emitter_table_dict_begin(emitter, "T1"); emitter_kv(emitter, "i4", "I4", emitter_type_int, &val); - emitter_json_dict_end(emitter); /* Close j1 */ + emitter_json_object_end(emitter); /* Close j1 */ emitter_kv(emitter, "i5", "I5", emitter_type_int, &val); emitter_table_dict_end(emitter); /* Close T1 */ emitter_kv(emitter, "i6", "I6", emitter_type_int, &val); @@ -302,24 +303,26 @@ emit_json_arr(emitter_t *emitter) { int ival = 123; emitter_begin(emitter); - emitter_json_dict_begin(emitter, "dict"); - emitter_json_arr_begin(emitter, "arr"); - emitter_json_arr_obj_begin(emitter); + emitter_json_key(emitter, "dict"); + emitter_json_object_begin(emitter); + emitter_json_key(emitter, "arr"); + emitter_json_array_begin(emitter); + emitter_json_object_begin(emitter); emitter_json_kv(emitter, "foo", emitter_type_int, &ival); - emitter_json_arr_obj_end(emitter); /* Close arr[0] */ + emitter_json_object_end(emitter); /* Close arr[0] */ /* arr[1] and arr[2] are primitives. */ - emitter_json_arr_value(emitter, emitter_type_int, &ival); - emitter_json_arr_value(emitter, emitter_type_int, &ival); - emitter_json_arr_obj_begin(emitter); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_object_begin(emitter); emitter_json_kv(emitter, "bar", emitter_type_int, &ival); emitter_json_kv(emitter, "baz", emitter_type_int, &ival); - emitter_json_arr_obj_end(emitter); /* Close arr[3]. */ - emitter_json_arr_end(emitter); /* Close arr. */ - emitter_json_dict_end(emitter); /* Close dict. */ + emitter_json_object_end(emitter); /* Close arr[3]. */ + emitter_json_array_end(emitter); /* Close arr. */ + emitter_json_object_end(emitter); /* Close dict. */ emitter_end(emitter); } -static const char *json_arr_json = +static const char *json_array_json = "{\n" "\t\"dict\": {\n" "\t\t\"arr\": [\n" @@ -336,10 +339,62 @@ static const char *json_arr_json = "\t}\n" "}\n"; -static const char *json_arr_table = ""; +static const char *json_array_table = ""; TEST_BEGIN(test_json_arr) { - assert_emit_output(&emit_json_arr, json_arr_json, json_arr_table); + assert_emit_output(&emit_json_arr, json_array_json, json_array_table); +} +TEST_END + +static void +emit_json_nested_array(emitter_t *emitter) { + int ival = 123; + char *sval = "foo"; + emitter_begin(emitter); + emitter_json_array_begin(emitter); + emitter_json_array_begin(emitter); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_value(emitter, emitter_type_string, &sval); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_value(emitter, emitter_type_string, &sval); + emitter_json_array_end(emitter); + emitter_json_array_begin(emitter); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_array_end(emitter); + emitter_json_array_begin(emitter); + emitter_json_value(emitter, emitter_type_string, &sval); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_array_end(emitter); + emitter_json_array_begin(emitter); + emitter_json_array_end(emitter); + emitter_json_array_end(emitter); + emitter_end(emitter); +} + +static const char *json_nested_array_json = +"{\n" +"\t[\n" +"\t\t[\n" +"\t\t\t123,\n" +"\t\t\t\"foo\",\n" +"\t\t\t123,\n" +"\t\t\t\"foo\"\n" +"\t\t],\n" +"\t\t[\n" +"\t\t\t123\n" +"\t\t],\n" +"\t\t[\n" +"\t\t\t\"foo\",\n" +"\t\t\t123\n" +"\t\t],\n" +"\t\t[\n" +"\t\t]\n" +"\t]\n" +"}\n"; + +TEST_BEGIN(test_json_nested_arr) { + assert_emit_output(&emit_json_nested_array, json_nested_array_json, + json_array_table); } TEST_END @@ -347,11 +402,11 @@ static void emit_table_row(emitter_t *emitter) { emitter_begin(emitter); emitter_row_t row; - emitter_col_t abc = {emitter_justify_left, 10, emitter_type_title}; + emitter_col_t abc = {emitter_justify_left, 10, emitter_type_title, {0}, {0, 0}}; abc.str_val = "ABC title"; - emitter_col_t def = {emitter_justify_right, 15, emitter_type_title}; + emitter_col_t def = {emitter_justify_right, 15, emitter_type_title, {0}, {0, 0}}; def.str_val = "DEF title"; - emitter_col_t ghi = {emitter_justify_right, 5, emitter_type_title}; + emitter_col_t ghi = {emitter_justify_right, 5, emitter_type_title, {0}, {0, 0}}; ghi.str_val = "GHI"; emitter_row_init(&row); @@ -409,5 +464,6 @@ main(void) { test_types, test_modal, test_json_arr, + test_json_nested_arr, test_table_row); } diff --git a/deps/jemalloc/test/unit/extent_util.c b/deps/jemalloc/test/unit/extent_util.c new file mode 100644 index 000000000..97e55f0f6 --- /dev/null +++ b/deps/jemalloc/test/unit/extent_util.c @@ -0,0 +1,269 @@ +#include "test/jemalloc_test.h" + +#define TEST_UTIL_EINVAL(node, a, b, c, d, why_inval) do { \ + assert_d_eq(mallctl("experimental.utilization." node, \ + a, b, c, d), EINVAL, "Should fail when " why_inval); \ + assert_zu_eq(out_sz, out_sz_ref, \ + "Output size touched when given invalid arguments"); \ + assert_d_eq(memcmp(out, out_ref, out_sz_ref), 0, \ + "Output content touched when given invalid arguments"); \ +} while (0) + +#define TEST_UTIL_QUERY_EINVAL(a, b, c, d, why_inval) \ + TEST_UTIL_EINVAL("query", a, b, c, d, why_inval) +#define TEST_UTIL_BATCH_EINVAL(a, b, c, d, why_inval) \ + TEST_UTIL_EINVAL("batch_query", a, b, c, d, why_inval) + +#define TEST_UTIL_VALID(node) do { \ + assert_d_eq(mallctl("experimental.utilization." node, \ + out, &out_sz, in, in_sz), 0, \ + "Should return 0 on correct arguments"); \ + assert_zu_eq(out_sz, out_sz_ref, "incorrect output size"); \ + assert_d_ne(memcmp(out, out_ref, out_sz_ref), 0, \ + "Output content should be changed"); \ +} while (0) + +#define TEST_UTIL_BATCH_VALID TEST_UTIL_VALID("batch_query") + +#define TEST_MAX_SIZE (1 << 20) + +TEST_BEGIN(test_query) { + size_t sz; + /* + * Select some sizes that can span both small and large sizes, and are + * numerically unrelated to any size boundaries. + */ + for (sz = 7; sz <= TEST_MAX_SIZE && sz <= SC_LARGE_MAXCLASS; + sz += (sz <= SC_SMALL_MAXCLASS ? 1009 : 99989)) { + void *p = mallocx(sz, 0); + void **in = &p; + size_t in_sz = sizeof(const void *); + size_t out_sz = sizeof(void *) + sizeof(size_t) * 5; + void *out = mallocx(out_sz, 0); + void *out_ref = mallocx(out_sz, 0); + size_t out_sz_ref = out_sz; + + assert_ptr_not_null(p, + "test pointer allocation failed"); + assert_ptr_not_null(out, + "test output allocation failed"); + assert_ptr_not_null(out_ref, + "test reference output allocation failed"); + +#define SLABCUR_READ(out) (*(void **)out) +#define COUNTS(out) ((size_t *)((void **)out + 1)) +#define NFREE_READ(out) COUNTS(out)[0] +#define NREGS_READ(out) COUNTS(out)[1] +#define SIZE_READ(out) COUNTS(out)[2] +#define BIN_NFREE_READ(out) COUNTS(out)[3] +#define BIN_NREGS_READ(out) COUNTS(out)[4] + + SLABCUR_READ(out) = NULL; + NFREE_READ(out) = NREGS_READ(out) = SIZE_READ(out) = -1; + BIN_NFREE_READ(out) = BIN_NREGS_READ(out) = -1; + memcpy(out_ref, out, out_sz); + + /* Test invalid argument(s) errors */ + TEST_UTIL_QUERY_EINVAL(NULL, &out_sz, in, in_sz, + "old is NULL"); + TEST_UTIL_QUERY_EINVAL(out, NULL, in, in_sz, + "oldlenp is NULL"); + TEST_UTIL_QUERY_EINVAL(out, &out_sz, NULL, in_sz, + "newp is NULL"); + TEST_UTIL_QUERY_EINVAL(out, &out_sz, in, 0, + "newlen is zero"); + in_sz -= 1; + TEST_UTIL_QUERY_EINVAL(out, &out_sz, in, in_sz, + "invalid newlen"); + in_sz += 1; + out_sz_ref = out_sz -= 2 * sizeof(size_t); + TEST_UTIL_QUERY_EINVAL(out, &out_sz, in, in_sz, + "invalid *oldlenp"); + out_sz_ref = out_sz += 2 * sizeof(size_t); + + /* Examine output for valid call */ + TEST_UTIL_VALID("query"); + assert_zu_le(sz, SIZE_READ(out), + "Extent size should be at least allocation size"); + assert_zu_eq(SIZE_READ(out) & (PAGE - 1), 0, + "Extent size should be a multiple of page size"); + if (sz <= SC_SMALL_MAXCLASS) { + assert_zu_le(NFREE_READ(out), NREGS_READ(out), + "Extent free count exceeded region count"); + assert_zu_le(NREGS_READ(out), SIZE_READ(out), + "Extent region count exceeded size"); + assert_zu_ne(NREGS_READ(out), 0, + "Extent region count must be positive"); + assert_ptr_not_null(SLABCUR_READ(out), + "Current slab is null"); + assert_true(NFREE_READ(out) == 0 + || SLABCUR_READ(out) <= p, + "Allocation should follow first fit principle"); + if (config_stats) { + assert_zu_le(BIN_NFREE_READ(out), + BIN_NREGS_READ(out), + "Bin free count exceeded region count"); + assert_zu_ne(BIN_NREGS_READ(out), 0, + "Bin region count must be positive"); + assert_zu_le(NFREE_READ(out), + BIN_NFREE_READ(out), + "Extent free count exceeded bin free count"); + assert_zu_le(NREGS_READ(out), + BIN_NREGS_READ(out), + "Extent region count exceeded " + "bin region count"); + assert_zu_eq(BIN_NREGS_READ(out) + % NREGS_READ(out), 0, + "Bin region count isn't a multiple of " + "extent region count"); + assert_zu_le( + BIN_NFREE_READ(out) - NFREE_READ(out), + BIN_NREGS_READ(out) - NREGS_READ(out), + "Free count in other extents in the bin " + "exceeded region count in other extents " + "in the bin"); + assert_zu_le(NREGS_READ(out) - NFREE_READ(out), + BIN_NREGS_READ(out) - BIN_NFREE_READ(out), + "Extent utilized count exceeded " + "bin utilized count"); + } + } else { + assert_zu_eq(NFREE_READ(out), 0, + "Extent free count should be zero"); + assert_zu_eq(NREGS_READ(out), 1, + "Extent region count should be one"); + assert_ptr_null(SLABCUR_READ(out), + "Current slab must be null for large size classes"); + if (config_stats) { + assert_zu_eq(BIN_NFREE_READ(out), 0, + "Bin free count must be zero for " + "large sizes"); + assert_zu_eq(BIN_NREGS_READ(out), 0, + "Bin region count must be zero for " + "large sizes"); + } + } + +#undef BIN_NREGS_READ +#undef BIN_NFREE_READ +#undef SIZE_READ +#undef NREGS_READ +#undef NFREE_READ +#undef COUNTS +#undef SLABCUR_READ + + free(out_ref); + free(out); + free(p); + } +} +TEST_END + +TEST_BEGIN(test_batch) { + size_t sz; + /* + * Select some sizes that can span both small and large sizes, and are + * numerically unrelated to any size boundaries. + */ + for (sz = 17; sz <= TEST_MAX_SIZE && sz <= SC_LARGE_MAXCLASS; + sz += (sz <= SC_SMALL_MAXCLASS ? 1019 : 99991)) { + void *p = mallocx(sz, 0); + void *q = mallocx(sz, 0); + void *in[] = {p, q}; + size_t in_sz = sizeof(const void *) * 2; + size_t out[] = {-1, -1, -1, -1, -1, -1}; + size_t out_sz = sizeof(size_t) * 6; + size_t out_ref[] = {-1, -1, -1, -1, -1, -1}; + size_t out_sz_ref = out_sz; + + assert_ptr_not_null(p, "test pointer allocation failed"); + assert_ptr_not_null(q, "test pointer allocation failed"); + + /* Test invalid argument(s) errors */ + TEST_UTIL_BATCH_EINVAL(NULL, &out_sz, in, in_sz, + "old is NULL"); + TEST_UTIL_BATCH_EINVAL(out, NULL, in, in_sz, + "oldlenp is NULL"); + TEST_UTIL_BATCH_EINVAL(out, &out_sz, NULL, in_sz, + "newp is NULL"); + TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, 0, + "newlen is zero"); + in_sz -= 1; + TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, in_sz, + "newlen is not an exact multiple"); + in_sz += 1; + out_sz_ref = out_sz -= 2 * sizeof(size_t); + TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, in_sz, + "*oldlenp is not an exact multiple"); + out_sz_ref = out_sz += 2 * sizeof(size_t); + in_sz -= sizeof(const void *); + TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, in_sz, + "*oldlenp and newlen do not match"); + in_sz += sizeof(const void *); + + /* Examine output for valid calls */ +#define TEST_EQUAL_REF(i, message) \ + assert_d_eq(memcmp(out + (i) * 3, out_ref + (i) * 3, 3), 0, message) + +#define NFREE_READ(out, i) out[(i) * 3] +#define NREGS_READ(out, i) out[(i) * 3 + 1] +#define SIZE_READ(out, i) out[(i) * 3 + 2] + + out_sz_ref = out_sz /= 2; + in_sz /= 2; + TEST_UTIL_BATCH_VALID; + assert_zu_le(sz, SIZE_READ(out, 0), + "Extent size should be at least allocation size"); + assert_zu_eq(SIZE_READ(out, 0) & (PAGE - 1), 0, + "Extent size should be a multiple of page size"); + if (sz <= SC_SMALL_MAXCLASS) { + assert_zu_le(NFREE_READ(out, 0), NREGS_READ(out, 0), + "Extent free count exceeded region count"); + assert_zu_le(NREGS_READ(out, 0), SIZE_READ(out, 0), + "Extent region count exceeded size"); + assert_zu_ne(NREGS_READ(out, 0), 0, + "Extent region count must be positive"); + } else { + assert_zu_eq(NFREE_READ(out, 0), 0, + "Extent free count should be zero"); + assert_zu_eq(NREGS_READ(out, 0), 1, + "Extent region count should be one"); + } + TEST_EQUAL_REF(1, + "Should not overwrite content beyond what's needed"); + in_sz *= 2; + out_sz_ref = out_sz *= 2; + + memcpy(out_ref, out, 3 * sizeof(size_t)); + TEST_UTIL_BATCH_VALID; + TEST_EQUAL_REF(0, "Statistics should be stable across calls"); + if (sz <= SC_SMALL_MAXCLASS) { + assert_zu_le(NFREE_READ(out, 1), NREGS_READ(out, 1), + "Extent free count exceeded region count"); + } else { + assert_zu_eq(NFREE_READ(out, 0), 0, + "Extent free count should be zero"); + } + assert_zu_eq(NREGS_READ(out, 0), NREGS_READ(out, 1), + "Extent region count should be same for same region size"); + assert_zu_eq(SIZE_READ(out, 0), SIZE_READ(out, 1), + "Extent size should be same for same region size"); + +#undef SIZE_READ +#undef NREGS_READ +#undef NFREE_READ + +#undef TEST_EQUAL_REF + + free(q); + free(p); + } +} +TEST_END + +int +main(void) { + assert_zu_lt(SC_SMALL_MAXCLASS, TEST_MAX_SIZE, + "Test case cannot cover large classes"); + return test(test_query, test_batch); +} diff --git a/deps/jemalloc/test/unit/hook.c b/deps/jemalloc/test/unit/hook.c new file mode 100644 index 000000000..72fcc433c --- /dev/null +++ b/deps/jemalloc/test/unit/hook.c @@ -0,0 +1,580 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/hook.h" + +static void *arg_extra; +static int arg_type; +static void *arg_result; +static void *arg_address; +static size_t arg_old_usize; +static size_t arg_new_usize; +static uintptr_t arg_result_raw; +static uintptr_t arg_args_raw[4]; + +static int call_count = 0; + +static void +reset_args() { + arg_extra = NULL; + arg_type = 12345; + arg_result = NULL; + arg_address = NULL; + arg_old_usize = 0; + arg_new_usize = 0; + arg_result_raw = 0; + memset(arg_args_raw, 77, sizeof(arg_args_raw)); +} + +static void +alloc_free_size(size_t sz) { + void *ptr = mallocx(1, 0); + free(ptr); + ptr = mallocx(1, 0); + free(ptr); + ptr = mallocx(1, MALLOCX_TCACHE_NONE); + dallocx(ptr, MALLOCX_TCACHE_NONE); +} + +/* + * We want to support a degree of user reentrancy. This tests a variety of + * allocation scenarios. + */ +static void +be_reentrant() { + /* Let's make sure the tcache is non-empty if enabled. */ + alloc_free_size(1); + alloc_free_size(1024); + alloc_free_size(64 * 1024); + alloc_free_size(256 * 1024); + alloc_free_size(1024 * 1024); + + /* Some reallocation. */ + void *ptr = mallocx(129, 0); + ptr = rallocx(ptr, 130, 0); + free(ptr); + + ptr = mallocx(2 * 1024 * 1024, 0); + free(ptr); + ptr = mallocx(1 * 1024 * 1024, 0); + ptr = rallocx(ptr, 2 * 1024 * 1024, 0); + free(ptr); + + ptr = mallocx(1, 0); + ptr = rallocx(ptr, 1000, 0); + free(ptr); +} + +static void +set_args_raw(uintptr_t *args_raw, int nargs) { + memcpy(arg_args_raw, args_raw, sizeof(uintptr_t) * nargs); +} + +static void +assert_args_raw(uintptr_t *args_raw_expected, int nargs) { + int cmp = memcmp(args_raw_expected, arg_args_raw, + sizeof(uintptr_t) * nargs); + assert_d_eq(cmp, 0, "Raw args mismatch"); +} + +static void +reset() { + call_count = 0; + reset_args(); +} + +static void +test_alloc_hook(void *extra, hook_alloc_t type, void *result, + uintptr_t result_raw, uintptr_t args_raw[3]) { + call_count++; + arg_extra = extra; + arg_type = (int)type; + arg_result = result; + arg_result_raw = result_raw; + set_args_raw(args_raw, 3); + be_reentrant(); +} + +static void +test_dalloc_hook(void *extra, hook_dalloc_t type, void *address, + uintptr_t args_raw[3]) { + call_count++; + arg_extra = extra; + arg_type = (int)type; + arg_address = address; + set_args_raw(args_raw, 3); + be_reentrant(); +} + +static void +test_expand_hook(void *extra, hook_expand_t type, void *address, + size_t old_usize, size_t new_usize, uintptr_t result_raw, + uintptr_t args_raw[4]) { + call_count++; + arg_extra = extra; + arg_type = (int)type; + arg_address = address; + arg_old_usize = old_usize; + arg_new_usize = new_usize; + arg_result_raw = result_raw; + set_args_raw(args_raw, 4); + be_reentrant(); +} + +TEST_BEGIN(test_hooks_basic) { + /* Just verify that the record their arguments correctly. */ + hooks_t hooks = { + &test_alloc_hook, &test_dalloc_hook, &test_expand_hook, + (void *)111}; + void *handle = hook_install(TSDN_NULL, &hooks); + uintptr_t args_raw[4] = {10, 20, 30, 40}; + + /* Alloc */ + reset_args(); + hook_invoke_alloc(hook_alloc_posix_memalign, (void *)222, 333, + args_raw); + assert_ptr_eq(arg_extra, (void *)111, "Passed wrong user pointer"); + assert_d_eq((int)hook_alloc_posix_memalign, arg_type, + "Passed wrong alloc type"); + assert_ptr_eq((void *)222, arg_result, "Passed wrong result address"); + assert_u64_eq(333, arg_result_raw, "Passed wrong result"); + assert_args_raw(args_raw, 3); + + /* Dalloc */ + reset_args(); + hook_invoke_dalloc(hook_dalloc_sdallocx, (void *)222, args_raw); + assert_d_eq((int)hook_dalloc_sdallocx, arg_type, + "Passed wrong dalloc type"); + assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer"); + assert_ptr_eq((void *)222, arg_address, "Passed wrong address"); + assert_args_raw(args_raw, 3); + + /* Expand */ + reset_args(); + hook_invoke_expand(hook_expand_xallocx, (void *)222, 333, 444, 555, + args_raw); + assert_d_eq((int)hook_expand_xallocx, arg_type, + "Passed wrong expand type"); + assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer"); + assert_ptr_eq((void *)222, arg_address, "Passed wrong address"); + assert_zu_eq(333, arg_old_usize, "Passed wrong old usize"); + assert_zu_eq(444, arg_new_usize, "Passed wrong new usize"); + assert_zu_eq(555, arg_result_raw, "Passed wrong result"); + assert_args_raw(args_raw, 4); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_null) { + /* Null hooks should be ignored, not crash. */ + hooks_t hooks1 = {NULL, NULL, NULL, NULL}; + hooks_t hooks2 = {&test_alloc_hook, NULL, NULL, NULL}; + hooks_t hooks3 = {NULL, &test_dalloc_hook, NULL, NULL}; + hooks_t hooks4 = {NULL, NULL, &test_expand_hook, NULL}; + + void *handle1 = hook_install(TSDN_NULL, &hooks1); + void *handle2 = hook_install(TSDN_NULL, &hooks2); + void *handle3 = hook_install(TSDN_NULL, &hooks3); + void *handle4 = hook_install(TSDN_NULL, &hooks4); + + assert_ptr_ne(handle1, NULL, "Hook installation failed"); + assert_ptr_ne(handle2, NULL, "Hook installation failed"); + assert_ptr_ne(handle3, NULL, "Hook installation failed"); + assert_ptr_ne(handle4, NULL, "Hook installation failed"); + + uintptr_t args_raw[4] = {10, 20, 30, 40}; + + call_count = 0; + hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw); + assert_d_eq(call_count, 1, "Called wrong number of times"); + + call_count = 0; + hook_invoke_dalloc(hook_dalloc_free, NULL, args_raw); + assert_d_eq(call_count, 1, "Called wrong number of times"); + + call_count = 0; + hook_invoke_expand(hook_expand_realloc, NULL, 0, 0, 0, args_raw); + assert_d_eq(call_count, 1, "Called wrong number of times"); + + hook_remove(TSDN_NULL, handle1); + hook_remove(TSDN_NULL, handle2); + hook_remove(TSDN_NULL, handle3); + hook_remove(TSDN_NULL, handle4); +} +TEST_END + +TEST_BEGIN(test_hooks_remove) { + hooks_t hooks = {&test_alloc_hook, NULL, NULL, NULL}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + call_count = 0; + uintptr_t args_raw[4] = {10, 20, 30, 40}; + hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw); + assert_d_eq(call_count, 1, "Hook not invoked"); + + call_count = 0; + hook_remove(TSDN_NULL, handle); + hook_invoke_alloc(hook_alloc_malloc, NULL, 0, NULL); + assert_d_eq(call_count, 0, "Hook invoked after removal"); + +} +TEST_END + +TEST_BEGIN(test_hooks_alloc_simple) { + /* "Simple" in the sense that we're not in a realloc variant. */ + hooks_t hooks = {&test_alloc_hook, NULL, NULL, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + /* Stop malloc from being optimized away. */ + volatile int err; + void *volatile ptr; + + /* malloc */ + reset(); + ptr = malloc(1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_malloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); + free(ptr); + + /* posix_memalign */ + reset(); + err = posix_memalign((void **)&ptr, 1024, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_posix_memalign, + "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)err, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)&ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1024, arg_args_raw[1], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[2], "Wrong argument"); + free(ptr); + + /* aligned_alloc */ + reset(); + ptr = aligned_alloc(1024, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_aligned_alloc, + "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1024, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* calloc */ + reset(); + ptr = calloc(11, 13); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_calloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)11, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)13, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* memalign */ +#ifdef JEMALLOC_OVERRIDE_MEMALIGN + reset(); + ptr = memalign(1024, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_memalign, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1024, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); + free(ptr); +#endif /* JEMALLOC_OVERRIDE_MEMALIGN */ + + /* valloc */ +#ifdef JEMALLOC_OVERRIDE_VALLOC + reset(); + ptr = valloc(1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_valloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); + free(ptr); +#endif /* JEMALLOC_OVERRIDE_VALLOC */ + + /* mallocx */ + reset(); + ptr = mallocx(1, MALLOCX_LG_ALIGN(10)); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_mallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)MALLOCX_LG_ALIGN(10), arg_args_raw[1], + "Wrong flags"); + free(ptr); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_dalloc_simple) { + /* "Simple" in the sense that we're not in a realloc variant. */ + hooks_t hooks = {NULL, &test_dalloc_hook, NULL, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + + /* free() */ + reset(); + ptr = malloc(1); + free(ptr); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_free, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + + /* dallocx() */ + reset(); + ptr = malloc(1); + dallocx(ptr, MALLOCX_TCACHE_NONE); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_dallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + assert_u64_eq((uintptr_t)MALLOCX_TCACHE_NONE, arg_args_raw[1], + "Wrong raw arg"); + + /* sdallocx() */ + reset(); + ptr = malloc(1); + sdallocx(ptr, 1, MALLOCX_TCACHE_NONE); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_sdallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong raw arg"); + assert_u64_eq((uintptr_t)MALLOCX_TCACHE_NONE, arg_args_raw[2], + "Wrong raw arg"); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_expand_simple) { + /* "Simple" in the sense that we're not in a realloc variant. */ + hooks_t hooks = {NULL, NULL, &test_expand_hook, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + + /* xallocx() */ + reset(); + ptr = malloc(1); + size_t new_usize = xallocx(ptr, 100, 200, MALLOCX_TCACHE_NONE); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_expand_xallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer expanded"); + assert_u64_eq(arg_old_usize, nallocx(1, 0), "Wrong old usize"); + assert_u64_eq(arg_new_usize, sallocx(ptr, 0), "Wrong new usize"); + assert_u64_eq(new_usize, arg_result_raw, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong arg"); + assert_u64_eq(100, arg_args_raw[1], "Wrong arg"); + assert_u64_eq(200, arg_args_raw[2], "Wrong arg"); + assert_u64_eq(MALLOCX_TCACHE_NONE, arg_args_raw[3], "Wrong arg"); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_realloc_as_malloc_or_free) { + hooks_t hooks = {&test_alloc_hook, &test_dalloc_hook, + &test_expand_hook, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + + /* realloc(NULL, size) as malloc */ + reset(); + ptr = realloc(NULL, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_realloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)NULL, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* realloc(ptr, 0) as free */ + ptr = malloc(1); + reset(); + realloc(ptr, 0); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_realloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + assert_u64_eq((uintptr_t)0, arg_args_raw[1], "Wrong raw arg"); + + /* realloc(NULL, 0) as malloc(0) */ + reset(); + ptr = realloc(NULL, 0); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_realloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)NULL, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)0, arg_args_raw[1], "Wrong argument"); + free(ptr); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +static void +do_realloc_test(void *(*ralloc)(void *, size_t, int), int flags, + int expand_type, int dalloc_type) { + hooks_t hooks = {&test_alloc_hook, &test_dalloc_hook, + &test_expand_hook, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + void *volatile ptr2; + + /* Realloc in-place, small. */ + ptr = malloc(129); + reset(); + ptr2 = ralloc(ptr, 130, flags); + assert_ptr_eq(ptr, ptr2, "Small realloc moved"); + + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, expand_type, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong address"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)130, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* + * Realloc in-place, large. Since we can't guarantee the large case + * across all platforms, we stay resilient to moving results. + */ + ptr = malloc(2 * 1024 * 1024); + free(ptr); + ptr2 = malloc(1 * 1024 * 1024); + reset(); + ptr = ralloc(ptr2, 2 * 1024 * 1024, flags); + /* ptr is the new address, ptr2 is the old address. */ + if (ptr == ptr2) { + assert_d_eq(call_count, 1, "Hook not called"); + assert_d_eq(arg_type, expand_type, "Wrong hook type"); + } else { + assert_d_eq(call_count, 2, "Wrong hooks called"); + assert_ptr_eq(ptr, arg_result, "Wrong address"); + assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); + } + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_ptr_eq(ptr2, arg_address, "Wrong address"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr2, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)2 * 1024 * 1024, arg_args_raw[1], + "Wrong argument"); + free(ptr); + + /* Realloc with move, small. */ + ptr = malloc(8); + reset(); + ptr2 = ralloc(ptr, 128, flags); + assert_ptr_ne(ptr, ptr2, "Small realloc didn't move"); + + assert_d_eq(call_count, 2, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong address"); + assert_ptr_eq(ptr2, arg_result, "Wrong address"); + assert_u64_eq((uintptr_t)ptr2, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)128, arg_args_raw[1], "Wrong argument"); + free(ptr2); + + /* Realloc with move, large. */ + ptr = malloc(1); + reset(); + ptr2 = ralloc(ptr, 2 * 1024 * 1024, flags); + assert_ptr_ne(ptr, ptr2, "Large realloc didn't move"); + + assert_d_eq(call_count, 2, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong address"); + assert_ptr_eq(ptr2, arg_result, "Wrong address"); + assert_u64_eq((uintptr_t)ptr2, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)2 * 1024 * 1024, arg_args_raw[1], + "Wrong argument"); + free(ptr2); + + hook_remove(TSDN_NULL, handle); +} + +static void * +realloc_wrapper(void *ptr, size_t size, UNUSED int flags) { + return realloc(ptr, size); +} + +TEST_BEGIN(test_hooks_realloc) { + do_realloc_test(&realloc_wrapper, 0, hook_expand_realloc, + hook_dalloc_realloc); +} +TEST_END + +TEST_BEGIN(test_hooks_rallocx) { + do_realloc_test(&rallocx, MALLOCX_TCACHE_NONE, hook_expand_rallocx, + hook_dalloc_rallocx); +} +TEST_END + +int +main(void) { + /* We assert on call counts. */ + return test_no_reentrancy( + test_hooks_basic, + test_hooks_null, + test_hooks_remove, + test_hooks_alloc_simple, + test_hooks_dalloc_simple, + test_hooks_expand_simple, + test_hooks_realloc_as_malloc_or_free, + test_hooks_realloc, + test_hooks_rallocx); +} diff --git a/deps/jemalloc/test/unit/huge.c b/deps/jemalloc/test/unit/huge.c new file mode 100644 index 000000000..ab72cf007 --- /dev/null +++ b/deps/jemalloc/test/unit/huge.c @@ -0,0 +1,108 @@ +#include "test/jemalloc_test.h" + +/* Threshold: 2 << 20 = 2097152. */ +const char *malloc_conf = "oversize_threshold:2097152"; + +#define HUGE_SZ (2 << 20) +#define SMALL_SZ (8) + +TEST_BEGIN(huge_bind_thread) { + unsigned arena1, arena2; + size_t sz = sizeof(unsigned); + + /* Bind to a manual arena. */ + assert_d_eq(mallctl("arenas.create", &arena1, &sz, NULL, 0), 0, + "Failed to create arena"); + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena1, + sizeof(arena1)), 0, "Fail to bind thread"); + + void *ptr = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(ptr, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_eq(arena1, arena2, "Wrong arena used after binding"); + dallocx(ptr, 0); + + /* Switch back to arena 0. */ + test_skip_if(have_percpu_arena && + PERCPU_ARENA_ENABLED(opt_percpu_arena)); + arena2 = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena2, + sizeof(arena2)), 0, "Fail to bind thread"); + ptr = mallocx(SMALL_SZ, MALLOCX_TCACHE_NONE); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_eq(arena2, 0, "Wrong arena used after binding"); + dallocx(ptr, MALLOCX_TCACHE_NONE); + + /* Then huge allocation should use the huge arena. */ + ptr = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(ptr, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena2, 0, "Wrong arena used after binding"); + assert_u_ne(arena1, arena2, "Wrong arena used after binding"); + dallocx(ptr, 0); +} +TEST_END + +TEST_BEGIN(huge_mallocx) { + unsigned arena1, arena2; + size_t sz = sizeof(unsigned); + + assert_d_eq(mallctl("arenas.create", &arena1, &sz, NULL, 0), 0, + "Failed to create arena"); + void *huge = mallocx(HUGE_SZ, MALLOCX_ARENA(arena1)); + assert_ptr_not_null(huge, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge, + sizeof(huge)), 0, "Unexpected mallctl() failure"); + assert_u_eq(arena1, arena2, "Wrong arena used for mallocx"); + dallocx(huge, MALLOCX_ARENA(arena1)); + + void *huge2 = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(huge, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge2, + sizeof(huge2)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena1, arena2, + "Huge allocation should not come from the manual arena."); + assert_u_ne(arena2, 0, + "Huge allocation should not come from the arena 0."); + dallocx(huge2, 0); +} +TEST_END + +TEST_BEGIN(huge_allocation) { + unsigned arena1, arena2; + + void *ptr = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(ptr, "Fail to allocate huge size"); + size_t sz = sizeof(unsigned); + assert_d_eq(mallctl("arenas.lookup", &arena1, &sz, &ptr, sizeof(ptr)), + 0, "Unexpected mallctl() failure"); + assert_u_gt(arena1, 0, "Huge allocation should not come from arena 0"); + dallocx(ptr, 0); + + ptr = mallocx(HUGE_SZ >> 1, 0); + assert_ptr_not_null(ptr, "Fail to allocate half huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena1, arena2, "Wrong arena used for half huge"); + dallocx(ptr, 0); + + ptr = mallocx(SMALL_SZ, MALLOCX_TCACHE_NONE); + assert_ptr_not_null(ptr, "Fail to allocate small size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena1, arena2, + "Huge and small should be from different arenas"); + dallocx(ptr, 0); +} +TEST_END + +int +main(void) { + return test( + huge_allocation, + huge_mallocx, + huge_bind_thread); +} diff --git a/deps/jemalloc/test/unit/junk.c b/deps/jemalloc/test/unit/junk.c index 243ced41e..57e3ad431 100644 --- a/deps/jemalloc/test/unit/junk.c +++ b/deps/jemalloc/test/unit/junk.c @@ -123,13 +123,13 @@ test_junk(size_t sz_min, size_t sz_max) { TEST_BEGIN(test_junk_small) { test_skip_if(!config_fill); - test_junk(1, SMALL_MAXCLASS-1); + test_junk(1, SC_SMALL_MAXCLASS - 1); } TEST_END TEST_BEGIN(test_junk_large) { test_skip_if(!config_fill); - test_junk(SMALL_MAXCLASS+1, (1U << (LG_LARGE_MINCLASS+1))); + test_junk(SC_SMALL_MAXCLASS + 1, (1U << (SC_LG_LARGE_MINCLASS + 1))); } TEST_END diff --git a/deps/jemalloc/test/unit/mallctl.c b/deps/jemalloc/test/unit/mallctl.c index 1ecbab08e..3a75ac040 100644 --- a/deps/jemalloc/test/unit/mallctl.c +++ b/deps/jemalloc/test/unit/mallctl.c @@ -1,5 +1,6 @@ #include "test/jemalloc_test.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/util.h" TEST_BEGIN(test_mallctl_errors) { @@ -158,11 +159,13 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(bool, abort, always); TEST_MALLCTL_OPT(bool, abort_conf, always); + TEST_MALLCTL_OPT(bool, confirm_conf, always); TEST_MALLCTL_OPT(const char *, metadata_thp, always); TEST_MALLCTL_OPT(bool, retain, always); TEST_MALLCTL_OPT(const char *, dss, always); TEST_MALLCTL_OPT(unsigned, narenas, always); TEST_MALLCTL_OPT(const char *, percpu_arena, always); + TEST_MALLCTL_OPT(size_t, oversize_threshold, always); TEST_MALLCTL_OPT(bool, background_thread, always); TEST_MALLCTL_OPT(ssize_t, dirty_decay_ms, always); TEST_MALLCTL_OPT(ssize_t, muzzy_decay_ms, always); @@ -340,6 +343,9 @@ TEST_BEGIN(test_thread_arena) { sz = sizeof(unsigned); assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); + if (opt_oversize_threshold != 0) { + narenas--; + } assert_u_eq(narenas, opt_narenas, "Number of arenas incorrect"); if (strcmp(opa, "disabled") == 0) { @@ -576,7 +582,7 @@ TEST_BEGIN(test_arena_i_retain_grow_limit) { assert_d_eq(mallctlbymib(mib, miblen, &default_limit, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); - assert_zu_eq(default_limit, sz_pind2sz(EXTENT_GROW_MAX_PIND), + assert_zu_eq(default_limit, SC_LARGE_MAXCLASS, "Unexpected default for retain_grow_limit"); new_limit = PAGE - 1; @@ -681,8 +687,8 @@ TEST_BEGIN(test_arenas_constants) { TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM); TEST_ARENAS_CONSTANT(size_t, page, PAGE); - TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS); - TEST_ARENAS_CONSTANT(unsigned, nlextents, NSIZES - NBINS); + TEST_ARENAS_CONSTANT(unsigned, nbins, SC_NBINS); + TEST_ARENAS_CONSTANT(unsigned, nlextents, SC_NSIZES - SC_NBINS); #undef TEST_ARENAS_CONSTANT } @@ -701,6 +707,7 @@ TEST_BEGIN(test_arenas_bin_constants) { TEST_ARENAS_BIN_CONSTANT(uint32_t, nregs, bin_infos[0].nregs); TEST_ARENAS_BIN_CONSTANT(size_t, slab_size, bin_infos[0].slab_size); + TEST_ARENAS_BIN_CONSTANT(uint32_t, nshards, bin_infos[0].n_shards); #undef TEST_ARENAS_BIN_CONSTANT } @@ -715,7 +722,8 @@ TEST_BEGIN(test_arenas_lextent_constants) { assert_zu_eq(name, expected, "Incorrect "#name" size"); \ } while (0) - TEST_ARENAS_LEXTENT_CONSTANT(size_t, size, LARGE_MINCLASS); + TEST_ARENAS_LEXTENT_CONSTANT(size_t, size, + SC_LARGE_MINCLASS); #undef TEST_ARENAS_LEXTENT_CONSTANT } @@ -773,6 +781,79 @@ TEST_BEGIN(test_stats_arenas) { } TEST_END +static void +alloc_hook(void *extra, UNUSED hook_alloc_t type, UNUSED void *result, + UNUSED uintptr_t result_raw, UNUSED uintptr_t args_raw[3]) { + *(bool *)extra = true; +} + +static void +dalloc_hook(void *extra, UNUSED hook_dalloc_t type, + UNUSED void *address, UNUSED uintptr_t args_raw[3]) { + *(bool *)extra = true; +} + +TEST_BEGIN(test_hooks) { + bool hook_called = false; + hooks_t hooks = {&alloc_hook, &dalloc_hook, NULL, &hook_called}; + void *handle = NULL; + size_t sz = sizeof(handle); + int err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, + sizeof(hooks)); + assert_d_eq(err, 0, "Hook installation failed"); + assert_ptr_ne(handle, NULL, "Hook installation gave null handle"); + void *ptr = mallocx(1, 0); + assert_true(hook_called, "Alloc hook not called"); + hook_called = false; + free(ptr); + assert_true(hook_called, "Free hook not called"); + + err = mallctl("experimental.hooks.remove", NULL, NULL, &handle, + sizeof(handle)); + assert_d_eq(err, 0, "Hook removal failed"); + hook_called = false; + ptr = mallocx(1, 0); + free(ptr); + assert_false(hook_called, "Hook called after removal"); +} +TEST_END + +TEST_BEGIN(test_hooks_exhaustion) { + bool hook_called = false; + hooks_t hooks = {&alloc_hook, &dalloc_hook, NULL, &hook_called}; + + void *handle; + void *handles[HOOK_MAX]; + size_t sz = sizeof(handle); + int err; + for (int i = 0; i < HOOK_MAX; i++) { + handle = NULL; + err = mallctl("experimental.hooks.install", &handle, &sz, + &hooks, sizeof(hooks)); + assert_d_eq(err, 0, "Error installation hooks"); + assert_ptr_ne(handle, NULL, "Got NULL handle"); + handles[i] = handle; + } + err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, + sizeof(hooks)); + assert_d_eq(err, EAGAIN, "Should have failed hook installation"); + for (int i = 0; i < HOOK_MAX; i++) { + err = mallctl("experimental.hooks.remove", NULL, NULL, + &handles[i], sizeof(handles[i])); + assert_d_eq(err, 0, "Hook removal failed"); + } + /* Insertion failed, but then we removed some; it should work now. */ + handle = NULL; + err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, + sizeof(hooks)); + assert_d_eq(err, 0, "Hook insertion failed"); + assert_ptr_ne(handle, NULL, "Got NULL handle"); + err = mallctl("experimental.hooks.remove", NULL, NULL, &handle, + sizeof(handle)); + assert_d_eq(err, 0, "Hook removal failed"); +} +TEST_END + int main(void) { return test( @@ -801,5 +882,7 @@ main(void) { test_arenas_lextent_constants, test_arenas_create, test_arenas_lookup, - test_stats_arenas); + test_stats_arenas, + test_hooks, + test_hooks_exhaustion); } diff --git a/deps/jemalloc/test/unit/prof_gdump.c b/deps/jemalloc/test/unit/prof_gdump.c index fcb434cb9..f7e0aac76 100644 --- a/deps/jemalloc/test/unit/prof_gdump.c +++ b/deps/jemalloc/test/unit/prof_gdump.c @@ -29,12 +29,12 @@ TEST_BEGIN(test_gdump) { prof_dump_open = prof_dump_open_intercept; did_prof_dump_open = false; - p = mallocx((1U << LG_LARGE_MINCLASS), 0); + p = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); did_prof_dump_open = false; - q = mallocx((1U << LG_LARGE_MINCLASS), 0); + q = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); @@ -45,7 +45,7 @@ TEST_BEGIN(test_gdump) { "Unexpected mallctl failure while disabling prof.gdump"); assert(gdump_old); did_prof_dump_open = false; - r = mallocx((1U << LG_LARGE_MINCLASS), 0); + r = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_false(did_prof_dump_open, "Unexpected profile dump"); @@ -56,7 +56,7 @@ TEST_BEGIN(test_gdump) { "Unexpected mallctl failure while enabling prof.gdump"); assert(!gdump_old); did_prof_dump_open = false; - s = mallocx((1U << LG_LARGE_MINCLASS), 0); + s = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); diff --git a/deps/jemalloc/test/unit/prof_log.c b/deps/jemalloc/test/unit/prof_log.c new file mode 100644 index 000000000..92fbd7cea --- /dev/null +++ b/deps/jemalloc/test/unit/prof_log.c @@ -0,0 +1,148 @@ +#include "test/jemalloc_test.h" + +#define N_PARAM 100 +#define N_THREADS 10 + +static void assert_rep() { + assert_b_eq(prof_log_rep_check(), false, "Rep check failed"); +} + +static void assert_log_empty() { + assert_zu_eq(prof_log_bt_count(), 0, + "The log has backtraces; it isn't empty"); + assert_zu_eq(prof_log_thr_count(), 0, + "The log has threads; it isn't empty"); + assert_zu_eq(prof_log_alloc_count(), 0, + "The log has allocations; it isn't empty"); +} + +void *buf[N_PARAM]; + +static void f() { + int i; + for (i = 0; i < N_PARAM; i++) { + buf[i] = malloc(100); + } + for (i = 0; i < N_PARAM; i++) { + free(buf[i]); + } +} + +TEST_BEGIN(test_prof_log_many_logs) { + int i; + + test_skip_if(!config_prof); + + for (i = 0; i < N_PARAM; i++) { + assert_b_eq(prof_log_is_logging(), false, + "Logging shouldn't have started yet"); + assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when starting logging"); + assert_b_eq(prof_log_is_logging(), true, + "Logging should be started by now"); + assert_log_empty(); + assert_rep(); + f(); + assert_zu_eq(prof_log_thr_count(), 1, "Wrong thread count"); + assert_rep(); + assert_b_eq(prof_log_is_logging(), true, + "Logging should still be on"); + assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when stopping logging"); + assert_b_eq(prof_log_is_logging(), false, + "Logging should have turned off"); + } +} +TEST_END + +thd_t thr_buf[N_THREADS]; + +static void *f_thread(void *unused) { + int i; + for (i = 0; i < N_PARAM; i++) { + void *p = malloc(100); + memset(p, 100, sizeof(char)); + free(p); + } + + return NULL; +} + +TEST_BEGIN(test_prof_log_many_threads) { + + test_skip_if(!config_prof); + + int i; + assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when starting logging"); + for (i = 0; i < N_THREADS; i++) { + thd_create(&thr_buf[i], &f_thread, NULL); + } + + for (i = 0; i < N_THREADS; i++) { + thd_join(thr_buf[i], NULL); + } + assert_zu_eq(prof_log_thr_count(), N_THREADS, + "Wrong number of thread entries"); + assert_rep(); + assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when stopping logging"); +} +TEST_END + +static void f3() { + void *p = malloc(100); + free(p); +} + +static void f1() { + void *p = malloc(100); + f3(); + free(p); +} + +static void f2() { + void *p = malloc(100); + free(p); +} + +TEST_BEGIN(test_prof_log_many_traces) { + + test_skip_if(!config_prof); + + assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when starting logging"); + int i; + assert_rep(); + assert_log_empty(); + for (i = 0; i < N_PARAM; i++) { + assert_rep(); + f1(); + assert_rep(); + f2(); + assert_rep(); + f3(); + assert_rep(); + } + /* + * There should be 8 total backtraces: two for malloc/free in f1(), two + * for malloc/free in f2(), two for malloc/free in f3(), and then two + * for malloc/free in f1()'s call to f3(). However compiler + * optimizations such as loop unrolling might generate more call sites. + * So >= 8 traces are expected. + */ + assert_zu_ge(prof_log_bt_count(), 8, + "Expect at least 8 backtraces given sample workload"); + assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when stopping logging"); +} +TEST_END + +int +main(void) { + prof_log_dummy_set(true); + return test_no_reentrancy( + test_prof_log_many_logs, + test_prof_log_many_traces, + test_prof_log_many_threads); +} diff --git a/deps/jemalloc/test/unit/prof_log.sh b/deps/jemalloc/test/unit/prof_log.sh new file mode 100644 index 000000000..8fcc7d8a7 --- /dev/null +++ b/deps/jemalloc/test/unit/prof_log.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,lg_prof_sample:0" +fi diff --git a/deps/jemalloc/test/unit/retained.c b/deps/jemalloc/test/unit/retained.c index d51a59811..7993fd3d9 100644 --- a/deps/jemalloc/test/unit/retained.c +++ b/deps/jemalloc/test/unit/retained.c @@ -107,6 +107,9 @@ TEST_BEGIN(test_retained) { atomic_store_u(&epoch, 0, ATOMIC_RELAXED); unsigned nthreads = ncpus * 2; + if (LG_SIZEOF_PTR < 3 && nthreads > 16) { + nthreads = 16; /* 32-bit platform could run out of vaddr. */ + } VARIABLE_ARRAY(thd_t, threads, nthreads); for (unsigned i = 0; i < nthreads; i++) { thd_create(&threads[i], thd_start, NULL); diff --git a/deps/jemalloc/test/unit/rtree.c b/deps/jemalloc/test/unit/rtree.c index 908100fac..90adca134 100644 --- a/deps/jemalloc/test/unit/rtree.c +++ b/deps/jemalloc/test/unit/rtree.c @@ -85,11 +85,11 @@ TEST_END TEST_BEGIN(test_rtree_extrema) { extent_t extent_a, extent_b; - extent_init(&extent_a, NULL, NULL, LARGE_MINCLASS, false, - sz_size2index(LARGE_MINCLASS), 0, extent_state_active, false, - false, true); - extent_init(&extent_b, NULL, NULL, 0, false, NSIZES, 0, - extent_state_active, false, false, true); + extent_init(&extent_a, NULL, NULL, SC_LARGE_MINCLASS, false, + sz_size2index(SC_LARGE_MINCLASS), 0, + extent_state_active, false, false, true, EXTENT_NOT_HEAD); + extent_init(&extent_b, NULL, NULL, 0, false, SC_NSIZES, 0, + extent_state_active, false, false, true, EXTENT_NOT_HEAD); tsdn_t *tsdn = tsdn_fetch(); @@ -125,8 +125,8 @@ TEST_BEGIN(test_rtree_bits) { PAGE + (((uintptr_t)1) << LG_PAGE) - 1}; extent_t extent; - extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, - extent_state_active, false, false, true); + extent_init(&extent, NULL, NULL, 0, false, SC_NSIZES, 0, + extent_state_active, false, false, true, EXTENT_NOT_HEAD); rtree_t *rtree = &test_rtree; rtree_ctx_t rtree_ctx; @@ -135,7 +135,7 @@ TEST_BEGIN(test_rtree_bits) { for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) { assert_false(rtree_write(tsdn, rtree, &rtree_ctx, keys[i], - &extent, NSIZES, false), + &extent, SC_NSIZES, false), "Unexpected rtree_write() failure"); for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, @@ -166,8 +166,8 @@ TEST_BEGIN(test_rtree_random) { rtree_ctx_data_init(&rtree_ctx); extent_t extent; - extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, - extent_state_active, false, false, true); + extent_init(&extent, NULL, NULL, 0, false, SC_NSIZES, 0, + extent_state_active, false, false, true, EXTENT_NOT_HEAD); assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); @@ -177,7 +177,8 @@ TEST_BEGIN(test_rtree_random) { &rtree_ctx, keys[i], false, true); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_lookup() failure"); - rtree_leaf_elm_write(tsdn, rtree, elm, &extent, NSIZES, false); + rtree_leaf_elm_write(tsdn, rtree, elm, &extent, SC_NSIZES, + false); assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, keys[i], true), &extent, "rtree_extent_read() should return previously set value"); diff --git a/deps/jemalloc/test/unit/safety_check.c b/deps/jemalloc/test/unit/safety_check.c new file mode 100644 index 000000000..bf4bd86d6 --- /dev/null +++ b/deps/jemalloc/test/unit/safety_check.c @@ -0,0 +1,156 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/safety_check.h" + +/* + * Note that we get called through safety_check.sh, which turns on sampling for + * everything. + */ + +bool fake_abort_called; +void fake_abort(const char *message) { + (void)message; + fake_abort_called = true; +} + +TEST_BEGIN(test_malloc_free_overflow) { + test_skip_if(!config_prof); + test_skip_if(!config_opt_safety_checks); + + safety_check_set_abort(&fake_abort); + /* Buffer overflow! */ + char* ptr = malloc(128); + ptr[128] = 0; + free(ptr); + safety_check_set_abort(NULL); + + assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); + fake_abort_called = false; +} +TEST_END + +TEST_BEGIN(test_mallocx_dallocx_overflow) { + test_skip_if(!config_prof); + test_skip_if(!config_opt_safety_checks); + + safety_check_set_abort(&fake_abort); + /* Buffer overflow! */ + char* ptr = mallocx(128, 0); + ptr[128] = 0; + dallocx(ptr, 0); + safety_check_set_abort(NULL); + + assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); + fake_abort_called = false; +} +TEST_END + +TEST_BEGIN(test_malloc_sdallocx_overflow) { + test_skip_if(!config_prof); + test_skip_if(!config_opt_safety_checks); + + safety_check_set_abort(&fake_abort); + /* Buffer overflow! */ + char* ptr = malloc(128); + ptr[128] = 0; + sdallocx(ptr, 128, 0); + safety_check_set_abort(NULL); + + assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); + fake_abort_called = false; +} +TEST_END + +TEST_BEGIN(test_realloc_overflow) { + test_skip_if(!config_prof); + test_skip_if(!config_opt_safety_checks); + + safety_check_set_abort(&fake_abort); + /* Buffer overflow! */ + char* ptr = malloc(128); + ptr[128] = 0; + ptr = realloc(ptr, 129); + safety_check_set_abort(NULL); + free(ptr); + + assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); + fake_abort_called = false; +} +TEST_END + +TEST_BEGIN(test_rallocx_overflow) { + test_skip_if(!config_prof); + test_skip_if(!config_opt_safety_checks); + + safety_check_set_abort(&fake_abort); + /* Buffer overflow! */ + char* ptr = malloc(128); + ptr[128] = 0; + ptr = rallocx(ptr, 129, 0); + safety_check_set_abort(NULL); + free(ptr); + + assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); + fake_abort_called = false; +} +TEST_END + +TEST_BEGIN(test_xallocx_overflow) { + test_skip_if(!config_prof); + test_skip_if(!config_opt_safety_checks); + + safety_check_set_abort(&fake_abort); + /* Buffer overflow! */ + char* ptr = malloc(128); + ptr[128] = 0; + size_t result = xallocx(ptr, 129, 0, 0); + assert_zu_eq(result, 128, ""); + free(ptr); + assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); + fake_abort_called = false; + safety_check_set_abort(NULL); +} +TEST_END + +TEST_BEGIN(test_realloc_no_overflow) { + char* ptr = malloc(128); + ptr = realloc(ptr, 256); + ptr[128] = 0; + ptr[255] = 0; + free(ptr); + + ptr = malloc(128); + ptr = realloc(ptr, 64); + ptr[63] = 0; + ptr[0] = 0; + free(ptr); +} +TEST_END + +TEST_BEGIN(test_rallocx_no_overflow) { + char* ptr = malloc(128); + ptr = rallocx(ptr, 256, 0); + ptr[128] = 0; + ptr[255] = 0; + free(ptr); + + ptr = malloc(128); + ptr = rallocx(ptr, 64, 0); + ptr[63] = 0; + ptr[0] = 0; + free(ptr); +} +TEST_END + +int +main(void) { + return test( + test_malloc_free_overflow, + test_mallocx_dallocx_overflow, + test_malloc_sdallocx_overflow, + test_realloc_overflow, + test_rallocx_overflow, + test_xallocx_overflow, + test_realloc_no_overflow, + test_rallocx_no_overflow); +} diff --git a/deps/jemalloc/test/unit/safety_check.sh b/deps/jemalloc/test/unit/safety_check.sh new file mode 100644 index 000000000..8fcc7d8a7 --- /dev/null +++ b/deps/jemalloc/test/unit/safety_check.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,lg_prof_sample:0" +fi diff --git a/deps/jemalloc/test/unit/sc.c b/deps/jemalloc/test/unit/sc.c new file mode 100644 index 000000000..bf51d8e59 --- /dev/null +++ b/deps/jemalloc/test/unit/sc.c @@ -0,0 +1,33 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_update_slab_size) { + sc_data_t data; + memset(&data, 0, sizeof(data)); + sc_data_init(&data); + sc_t *tiny = &data.sc[0]; + size_t tiny_size = (ZU(1) << tiny->lg_base) + + (ZU(tiny->ndelta) << tiny->lg_delta); + size_t pgs_too_big = (tiny_size * BITMAP_MAXBITS + PAGE - 1) / PAGE + 1; + sc_data_update_slab_size(&data, tiny_size, tiny_size, (int)pgs_too_big); + assert_zu_lt((size_t)tiny->pgs, pgs_too_big, "Allowed excessive pages"); + + sc_data_update_slab_size(&data, 1, 10 * PAGE, 1); + for (int i = 0; i < data.nbins; i++) { + sc_t *sc = &data.sc[i]; + size_t reg_size = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + if (reg_size <= PAGE) { + assert_d_eq(sc->pgs, 1, "Ignored valid page size hint"); + } else { + assert_d_gt(sc->pgs, 1, + "Allowed invalid page size hint"); + } + } +} +TEST_END + +int +main(void) { + return test( + test_update_slab_size); +} diff --git a/deps/jemalloc/test/unit/seq.c b/deps/jemalloc/test/unit/seq.c new file mode 100644 index 000000000..19613b0b2 --- /dev/null +++ b/deps/jemalloc/test/unit/seq.c @@ -0,0 +1,95 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/seq.h" + +typedef struct data_s data_t; +struct data_s { + int arr[10]; +}; + +static void +set_data(data_t *data, int num) { + for (int i = 0; i < 10; i++) { + data->arr[i] = num; + } +} + +static void +assert_data(data_t *data) { + int num = data->arr[0]; + for (int i = 0; i < 10; i++) { + assert_d_eq(num, data->arr[i], "Data consistency error"); + } +} + +seq_define(data_t, data) + +typedef struct thd_data_s thd_data_t; +struct thd_data_s { + seq_data_t data; +}; + +static void * +seq_reader_thd(void *arg) { + thd_data_t *thd_data = (thd_data_t *)arg; + int iter = 0; + data_t local_data; + while (iter < 1000 * 1000 - 1) { + bool success = seq_try_load_data(&local_data, &thd_data->data); + if (success) { + assert_data(&local_data); + assert_d_le(iter, local_data.arr[0], + "Seq read went back in time."); + iter = local_data.arr[0]; + } + } + return NULL; +} + +static void * +seq_writer_thd(void *arg) { + thd_data_t *thd_data = (thd_data_t *)arg; + data_t local_data; + memset(&local_data, 0, sizeof(local_data)); + for (int i = 0; i < 1000 * 1000; i++) { + set_data(&local_data, i); + seq_store_data(&thd_data->data, &local_data); + } + return NULL; +} + +TEST_BEGIN(test_seq_threaded) { + thd_data_t thd_data; + memset(&thd_data, 0, sizeof(thd_data)); + + thd_t reader; + thd_t writer; + + thd_create(&reader, seq_reader_thd, &thd_data); + thd_create(&writer, seq_writer_thd, &thd_data); + + thd_join(reader, NULL); + thd_join(writer, NULL); +} +TEST_END + +TEST_BEGIN(test_seq_simple) { + data_t data; + seq_data_t seq; + memset(&seq, 0, sizeof(seq)); + for (int i = 0; i < 1000 * 1000; i++) { + set_data(&data, i); + seq_store_data(&seq, &data); + set_data(&data, 0); + bool success = seq_try_load_data(&data, &seq); + assert_b_eq(success, true, "Failed non-racing read"); + assert_data(&data); + } +} +TEST_END + +int main(void) { + return test_no_reentrancy( + test_seq_simple, + test_seq_threaded); +} diff --git a/deps/jemalloc/test/unit/size_classes.c b/deps/jemalloc/test/unit/size_classes.c index bcff56098..694733635 100644 --- a/deps/jemalloc/test/unit/size_classes.c +++ b/deps/jemalloc/test/unit/size_classes.c @@ -108,8 +108,13 @@ TEST_BEGIN(test_psize_classes) { size_class, sz_psz2ind(size_class), sz_pind2sz(sz_psz2ind(size_class))); - assert_u_eq(pind+1, sz_psz2ind(size_class+1), - "Next size_class does not round up properly"); + if (size_class == SC_LARGE_MAXCLASS) { + assert_u_eq(SC_NPSIZES, sz_psz2ind(size_class + 1), + "Next size_class does not round up properly"); + } else { + assert_u_eq(pind + 1, sz_psz2ind(size_class + 1), + "Next size_class does not round up properly"); + } assert_zu_eq(size_class, (pind > 0) ? sz_psz2u(sz_pind2sz(pind-1)+1) : sz_psz2u(1), @@ -142,11 +147,11 @@ TEST_BEGIN(test_overflow) { max_size_class = get_max_size_class(); max_psz = max_size_class + PAGE; - assert_u_eq(sz_size2index(max_size_class+1), NSIZES, + assert_u_eq(sz_size2index(max_size_class+1), SC_NSIZES, "sz_size2index() should return NSIZES on overflow"); - assert_u_eq(sz_size2index(ZU(PTRDIFF_MAX)+1), NSIZES, + assert_u_eq(sz_size2index(ZU(PTRDIFF_MAX)+1), SC_NSIZES, "sz_size2index() should return NSIZES on overflow"); - assert_u_eq(sz_size2index(SIZE_T_MAX), NSIZES, + assert_u_eq(sz_size2index(SIZE_T_MAX), SC_NSIZES, "sz_size2index() should return NSIZES on overflow"); assert_zu_eq(sz_s2u(max_size_class+1), 0, @@ -156,11 +161,11 @@ TEST_BEGIN(test_overflow) { assert_zu_eq(sz_s2u(SIZE_T_MAX), 0, "sz_s2u() should return 0 on overflow"); - assert_u_eq(sz_psz2ind(max_size_class+1), NPSIZES, + assert_u_eq(sz_psz2ind(max_size_class+1), SC_NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); - assert_u_eq(sz_psz2ind(ZU(PTRDIFF_MAX)+1), NPSIZES, + assert_u_eq(sz_psz2ind(ZU(PTRDIFF_MAX)+1), SC_NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); - assert_u_eq(sz_psz2ind(SIZE_T_MAX), NPSIZES, + assert_u_eq(sz_psz2ind(SIZE_T_MAX), SC_NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); assert_zu_eq(sz_psz2u(max_size_class+1), max_psz, diff --git a/deps/jemalloc/test/unit/slab.c b/deps/jemalloc/test/unit/slab.c index 7e662aed1..c56af25fe 100644 --- a/deps/jemalloc/test/unit/slab.c +++ b/deps/jemalloc/test/unit/slab.c @@ -3,13 +3,14 @@ TEST_BEGIN(test_arena_slab_regind) { szind_t binind; - for (binind = 0; binind < NBINS; binind++) { + for (binind = 0; binind < SC_NBINS; binind++) { size_t regind; extent_t slab; const bin_info_t *bin_info = &bin_infos[binind]; extent_init(&slab, NULL, mallocx(bin_info->slab_size, MALLOCX_LG_ALIGN(LG_PAGE)), bin_info->slab_size, true, - binind, 0, extent_state_active, false, true, true); + binind, 0, extent_state_active, false, true, true, + EXTENT_NOT_HEAD); assert_ptr_not_null(extent_addr_get(&slab), "Unexpected malloc() failure"); for (regind = 0; regind < bin_info->nregs; regind++) { diff --git a/deps/jemalloc/test/unit/stats.c b/deps/jemalloc/test/unit/stats.c index 231010e43..646768e88 100644 --- a/deps/jemalloc/test/unit/stats.c +++ b/deps/jemalloc/test/unit/stats.c @@ -33,7 +33,7 @@ TEST_BEGIN(test_stats_large) { size_t sz; int expected = config_stats ? 0 : ENOENT; - p = mallocx(SMALL_MAXCLASS+1, MALLOCX_ARENA(0)); + p = mallocx(SC_SMALL_MAXCLASS + 1, MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), @@ -74,9 +74,10 @@ TEST_BEGIN(test_stats_arenas_summary) { uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; - little = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); + little = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0)); assert_ptr_not_null(little, "Unexpected mallocx() failure"); - large = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); + large = mallocx((1U << SC_LG_LARGE_MINCLASS), + MALLOCX_ARENA(0)); assert_ptr_not_null(large, "Unexpected mallocx() failure"); dallocx(little, 0); @@ -148,7 +149,7 @@ TEST_BEGIN(test_stats_arenas_small) { no_lazy_lock(); /* Lazy locking would dodge tcache testing. */ - p = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); + p = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), @@ -191,7 +192,7 @@ TEST_BEGIN(test_stats_arenas_large) { uint64_t epoch, nmalloc, ndalloc; int expected = config_stats ? 0 : ENOENT; - p = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); + p = mallocx((1U << SC_LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), @@ -227,7 +228,7 @@ gen_mallctl_str(char *cmd, char *name, unsigned arena_ind) { TEST_BEGIN(test_stats_arenas_bins) { void *p; - size_t sz, curslabs, curregs; + size_t sz, curslabs, curregs, nonfull_slabs; uint64_t epoch, nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t nslabs, nreslabs; int expected = config_stats ? 0 : ENOENT; @@ -288,6 +289,9 @@ TEST_BEGIN(test_stats_arenas_bins) { gen_mallctl_str(cmd, "curslabs", arena_ind); assert_d_eq(mallctl(cmd, (void *)&curslabs, &sz, NULL, 0), expected, "Unexpected mallctl() result"); + gen_mallctl_str(cmd, "nonfull_slabs", arena_ind); + assert_d_eq(mallctl(cmd, (void *)&nonfull_slabs, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); if (config_stats) { assert_u64_gt(nmalloc, 0, @@ -308,6 +312,8 @@ TEST_BEGIN(test_stats_arenas_bins) { "At least one slab should have been allocated"); assert_zu_gt(curslabs, 0, "At least one slab should be currently allocated"); + assert_zu_eq(nonfull_slabs, 0, + "slabs_nonfull should be empty"); } dallocx(p, 0); diff --git a/deps/jemalloc/test/unit/hooks.c b/deps/jemalloc/test/unit/test_hooks.c similarity index 82% rename from deps/jemalloc/test/unit/hooks.c rename to deps/jemalloc/test/unit/test_hooks.c index b70172e13..ded8698bc 100644 --- a/deps/jemalloc/test/unit/hooks.c +++ b/deps/jemalloc/test/unit/test_hooks.c @@ -12,10 +12,10 @@ func_to_hook(int arg1, int arg2) { return arg1 + arg2; } -#define func_to_hook JEMALLOC_HOOK(func_to_hook, hooks_libc_hook) +#define func_to_hook JEMALLOC_HOOK(func_to_hook, test_hooks_libc_hook) TEST_BEGIN(unhooked_call) { - hooks_libc_hook = NULL; + test_hooks_libc_hook = NULL; hook_called = false; assert_d_eq(3, func_to_hook(1, 2), "Hooking changed return value."); assert_false(hook_called, "Nulling out hook didn't take."); @@ -23,7 +23,7 @@ TEST_BEGIN(unhooked_call) { TEST_END TEST_BEGIN(hooked_call) { - hooks_libc_hook = &hook; + test_hooks_libc_hook = &hook; hook_called = false; assert_d_eq(3, func_to_hook(1, 2), "Hooking changed return value."); assert_true(hook_called, "Hook should have executed."); diff --git a/deps/jemalloc/test/unit/tsd.c b/deps/jemalloc/test/unit/tsd.c index 6c479139b..917884dcf 100644 --- a/deps/jemalloc/test/unit/tsd.c +++ b/deps/jemalloc/test/unit/tsd.c @@ -1,5 +1,10 @@ #include "test/jemalloc_test.h" +/* + * If we're e.g. in debug mode, we *never* enter the fast path, and so shouldn't + * be asserting that we're on one. + */ +static bool originally_fast; static int data_cleanup_count; void @@ -98,11 +103,11 @@ thd_start_reincarnated(void *arg) { tsd_cleanup((void *)tsd); assert_ptr_null(*tsd_arenap_get_unsafe(tsd), "TSD arena should have been cleared."); - assert_u_eq(tsd->state, tsd_state_purgatory, + assert_u_eq(tsd_state_get(tsd), tsd_state_purgatory, "TSD state should be purgatory\n"); free(p); - assert_u_eq(tsd->state, tsd_state_reincarnated, + assert_u_eq(tsd_state_get(tsd), tsd_state_reincarnated, "TSD state should be reincarnated\n"); p = mallocx(1, MALLOCX_TCACHE_NONE); assert_ptr_not_null(p, "Unexpected malloc() failure"); @@ -124,6 +129,128 @@ TEST_BEGIN(test_tsd_reincarnation) { } TEST_END +typedef struct { + atomic_u32_t phase; + atomic_b_t error; +} global_slow_data_t; + +static void * +thd_start_global_slow(void *arg) { + /* PHASE 0 */ + global_slow_data_t *data = (global_slow_data_t *)arg; + free(mallocx(1, 0)); + + tsd_t *tsd = tsd_fetch(); + /* + * No global slowness has happened yet; there was an error if we were + * originally fast but aren't now. + */ + atomic_store_b(&data->error, originally_fast && !tsd_fast(tsd), + ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 1, ATOMIC_SEQ_CST); + + /* PHASE 2 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 2) { + } + free(mallocx(1, 0)); + atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 3, ATOMIC_SEQ_CST); + + /* PHASE 4 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 4) { + } + free(mallocx(1, 0)); + atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 5, ATOMIC_SEQ_CST); + + /* PHASE 6 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 6) { + } + free(mallocx(1, 0)); + /* Only one decrement so far. */ + atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 7, ATOMIC_SEQ_CST); + + /* PHASE 8 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 8) { + } + free(mallocx(1, 0)); + /* + * Both decrements happened; we should be fast again (if we ever + * were) + */ + atomic_store_b(&data->error, originally_fast && !tsd_fast(tsd), + ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 9, ATOMIC_SEQ_CST); + + return NULL; +} + +TEST_BEGIN(test_tsd_global_slow) { + global_slow_data_t data = {ATOMIC_INIT(0), ATOMIC_INIT(false)}; + /* + * Note that the "mallocx" here (vs. malloc) is important, since the + * compiler is allowed to optimize away free(malloc(1)) but not + * free(mallocx(1)). + */ + free(mallocx(1, 0)); + tsd_t *tsd = tsd_fetch(); + originally_fast = tsd_fast(tsd); + + thd_t thd; + thd_create(&thd, thd_start_global_slow, (void *)&data.phase); + /* PHASE 1 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 1) { + /* + * We don't have a portable condvar/semaphore mechanism. + * Spin-wait. + */ + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + tsd_global_slow_inc(tsd_tsdn(tsd)); + free(mallocx(1, 0)); + assert_false(tsd_fast(tsd), ""); + atomic_store_u32(&data.phase, 2, ATOMIC_SEQ_CST); + + /* PHASE 3 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 3) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + /* Increase again, so that we can test multiple fast/slow changes. */ + tsd_global_slow_inc(tsd_tsdn(tsd)); + atomic_store_u32(&data.phase, 4, ATOMIC_SEQ_CST); + free(mallocx(1, 0)); + assert_false(tsd_fast(tsd), ""); + + /* PHASE 5 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 5) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + tsd_global_slow_dec(tsd_tsdn(tsd)); + atomic_store_u32(&data.phase, 6, ATOMIC_SEQ_CST); + /* We only decreased once; things should still be slow. */ + free(mallocx(1, 0)); + assert_false(tsd_fast(tsd), ""); + + /* PHASE 7 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 7) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + tsd_global_slow_dec(tsd_tsdn(tsd)); + atomic_store_u32(&data.phase, 8, ATOMIC_SEQ_CST); + /* We incremented and then decremented twice; we should be fast now. */ + free(mallocx(1, 0)); + assert_true(!originally_fast || tsd_fast(tsd), ""); + + /* PHASE 9 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 9) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + + thd_join(thd, NULL); +} +TEST_END + int main(void) { /* Ensure tsd bootstrapped. */ @@ -135,5 +262,6 @@ main(void) { return test_no_reentrancy( test_tsd_main_thread, test_tsd_sub_thread, - test_tsd_reincarnation); + test_tsd_reincarnation, + test_tsd_global_slow); } diff --git a/deps/jemalloc/test/unit/zero.c b/deps/jemalloc/test/unit/zero.c index 553692ba7..271fd5cba 100644 --- a/deps/jemalloc/test/unit/zero.c +++ b/deps/jemalloc/test/unit/zero.c @@ -41,13 +41,13 @@ test_zero(size_t sz_min, size_t sz_max) { TEST_BEGIN(test_zero_small) { test_skip_if(!config_fill); - test_zero(1, SMALL_MAXCLASS-1); + test_zero(1, SC_SMALL_MAXCLASS - 1); } TEST_END TEST_BEGIN(test_zero_large) { test_skip_if(!config_fill); - test_zero(SMALL_MAXCLASS+1, (1U << (LG_LARGE_MINCLASS+1))); + test_zero(SC_SMALL_MAXCLASS + 1, 1U << (SC_LG_LARGE_MINCLASS + 1)); } TEST_END diff --git a/deps/lua/src/lauxlib.c b/deps/lua/src/lauxlib.c index 10f14e2c0..751f1e887 100644 --- a/deps/lua/src/lauxlib.c +++ b/deps/lua/src/lauxlib.c @@ -575,7 +575,7 @@ LUALIB_API int luaL_loadfile (lua_State *L, const char *filename) { if (lf.f == NULL) return errfile(L, "reopen", fnameindex); /* skip eventual `#!...' */ while ((c = getc(lf.f)) != EOF && c != LUA_SIGNATURE[0]) ; - lf.extraline = 0; + lf.extraline = 0; } ungetc(c, lf.f); status = lua_load(L, getF, &lf, lua_tostring(L, -1)); diff --git a/deps/lua/src/ldo.c b/deps/lua/src/ldo.c index 939940a4c..e641dfcd9 100644 --- a/deps/lua/src/ldo.c +++ b/deps/lua/src/ldo.c @@ -493,7 +493,7 @@ static void f_parser (lua_State *L, void *ud) { Proto *tf; Closure *cl; struct SParser *p = cast(struct SParser *, ud); - int c = luaZ_lookahead(p->z); + luaZ_lookahead(p->z); luaC_checkGC(L); tf = (luaY_parser)(L, p->z, &p->buff, p->name); diff --git a/deps/lua/src/ltablib.c b/deps/lua/src/ltablib.c index b6d9cb4ac..0bdac7f6c 100644 --- a/deps/lua/src/ltablib.c +++ b/deps/lua/src/ltablib.c @@ -137,7 +137,7 @@ static void addfield (lua_State *L, luaL_Buffer *b, int i) { if (!lua_isstring(L, -1)) luaL_error(L, "invalid value (%s) at index %d in table for " LUA_QL("concat"), luaL_typename(L, -1), i); - luaL_addvalue(b); + luaL_addvalue(b); } diff --git a/pkg/deb/conf/keydb.conf b/pkg/deb/conf/keydb.conf index 821591b32..f0768db28 100644 --- a/pkg/deb/conf/keydb.conf +++ b/pkg/deb/conf/keydb.conf @@ -3,7 +3,7 @@ # Note that in order to read the configuration file, Redis must be # started with the file path as first argument: # -# ./keydb-server /path/to/redis.conf +# ./keydb-server /path/to/keydb.conf # Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: @@ -24,7 +24,7 @@ # to customize a few per-server settings. Include files can include # other files, so use this wisely. # -# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# Note that option "include" won't be rewritten by command "CONFIG REWRITE" # from admin or Redis Sentinel. Since Redis always uses the last processed # line as value of a configuration directive, you'd better put includes # at the beginning of this file to avoid overwriting config change at runtime. @@ -46,25 +46,31 @@ ################################## NETWORK ##################################### # By default, if no "bind" configuration directive is specified, Redis listens -# for connections from all the network interfaces available on the server. +# for connections from all available network interfaces on the host machine. # It is possible to listen to just one or multiple selected interfaces using # the "bind" configuration directive, followed by one or more IP addresses. +# Each address can be prefixed by "-", which means that KeyDB will not fail to +# start if the address is not available. Being not available only refers to +# addresses that does not correspond to any network interfece. Addresses that +# are already in use will always fail, and unsupported protocols will always BE +# silently skipped. # # Examples: # -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 ::1 +# bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses +# bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 +# bind * -::* # like the default, all available interfaces # # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the # internet, binding to all the interfaces is dangerous and will expose the # instance to everybody on the internet. So by default we uncomment the -# following bind directive, that will force Redis to listen only into -# the IPv4 loopback interface address (this means Redis will be able to -# accept connections only from clients running into the same computer it -# is running). +# following bind directive, that will force Redis to listen only on the +# IPv4 and IPv6 (if available) loopback interface addresses (this means Redis +# will only be able to accept client connections from the same host that it is +# running on). # # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES -# JUST COMMENT THE FOLLOWING LINE. +# JUST COMMENT OUT THE FOLLOWING LINE. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bind 127.0.0.1 ::1 @@ -93,8 +99,8 @@ port 6379 # TCP listen() backlog. # -# In high requests-per-second environments you need an high backlog in order -# to avoid slow clients connections issues. Note that the Linux kernel +# In high requests-per-second environments you need a high backlog in order +# to avoid slow clients connection issues. Note that the Linux kernel # will silently truncate it to the value of /proc/sys/net/core/somaxconn so # make sure to raise both the value of somaxconn and tcp_max_syn_backlog # in order to get the desired effect. @@ -106,7 +112,7 @@ tcp-backlog 511 # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # -# unixsocket /tmp/redis.sock +# unixsocket /run/keydb.sock # unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) @@ -118,8 +124,8 @@ timeout 0 # of communication. This is useful for two reasons: # # 1) Detect dead peers. -# 2) Take the connection alive from the point of view of network -# equipment in the middle. +# 2) Force network equipment in the middle to consider the connection to be +# alive. # # On Linux, the specified value (in seconds) is the period used to send ACKs. # Note that to close the connection the double of the time is needed. @@ -129,33 +135,158 @@ timeout 0 # Redis default starting with Redis 3.2.1. tcp-keepalive 300 +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file keydb.crt +# tls-key-file keydb.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret + +# Normally Redis uses the same certificate for both server functions (accepting +# connections) and client functions (replicating from a master, establishing +# cluster bus connections, etc.). +# +# Sometimes certificates are issued with attributes that designate them as +# client-only or server-only certificates. In that case it may be desired to use +# different certificates for incoming (server) and outgoing (client) +# connections. To do that, use the following directives: +# +# tls-client-cert-file client.crt +# tls-client-key-file client.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-client-key-file-pass secret + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file keydb.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: +# +# tls-cluster yes + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + ################################# GENERAL ##################################### # By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. -daemonize yes +# Note that Redis will write a pid file in /var/run/keydb.pid when daemonized. +# When Redis is supervised by upstart or systemd, this parameter has no impact. +daemonize no # If you run Redis from upstart or systemd, Redis can interact with your # supervision tree. Options: # supervised no - no supervision interaction # supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# requires "expect stop" in your upstart job config # supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# on startup, and updating Redis status on a regular +# basis. # supervised auto - detect upstart or systemd method based on # UPSTART_JOB or NOTIFY_SOCKET environment variables # Note: these supervision methods only signal "process is ready." -# They do not enable continuous liveness pings back to your supervisor. -supervised no +# They do not enable continuous pings back to your supervisor. +# +# The default is "no". To run under upstart/systemd, you can simply uncomment +# the line below: +# +# supervised auto # If a pid file is specified, Redis writes it where specified at startup # and removes it at exit. # # When the server runs non daemonized, no pid file is created if none is # specified in the configuration. When the server is daemonized, the pid file -# is used even if not specified, defaulting to "/var/run/redis.pid". +# is used even if not specified, defaulting to "/var/run/keydb.pid". # # Creating a pid file is best effort: if Redis is not able to create it # nothing bad happens, the server will start and run normally. -pidfile /var/run/keydb/keydb-server.pid +# +# Note that on modern Linux systems "/run/keydb.pid" is more conforming +# and should be used instead. +pidfile /var/run/keydb_6379.pid # Specify the server verbosity level. # This can be one of: @@ -175,49 +306,81 @@ logfile /var/log/keydb/keydb-server.log # syslog-enabled no # Specify the syslog identity. -# syslog-ident redis +# syslog-ident keydb # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. # syslog-facility local0 +# To disable the built in crash log, which will possibly produce cleaner core +# dumps when they are needed, uncomment the following: +# +# crash-log-enabled no + +# To disable the fast memory check that's run as part of the crash log, which +# will possibly let keydb terminate sooner, uncomment the following: +# +# crash-memcheck-enabled no + # Set the number of databases. The default database is DB 0, you can select # a different one on a per-connection basis using SELECT where # dbid is a number between 0 and 'databases'-1 databases 16 # By default Redis shows an ASCII art logo only when started to log to the -# standard output and if the standard output is a TTY. Basically this means -# that normally a logo is displayed only in interactive sessions. +# standard output and if the standard output is a TTY and syslog logging is +# disabled. Basically this means that normally a logo is displayed only in +# interactive sessions. # # However it is possible to force the pre-4.0 behavior and always show a # ASCII art logo in startup logs by setting the following option to yes. -always-show-logo yes +always-show-logo no + +# By default, Redis modifies the process title (as seen in 'top' and 'ps') to +# provide some runtime information. It is possible to disable this and leave +# the process name as executed by setting the following to no. +set-proc-title yes + +# When changing the process title, Redis uses the following template to construct +# the modified title. +# +# Template variables are specified in curly brackets. The following variables are +# supported: +# +# {title} Name of process as executed if parent, or type of child process. +# {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or +# Unix socket if only that's available. +# {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". +# {port} TCP port listening on, or 0. +# {tls-port} TLS port listening on, or 0. +# {unixsocket} Unix domain socket listening on, or "". +# {config-file} Name of configuration file used. +# +proc-title-template "{title} {listen-addr} {server-mode}" ################################ SNAPSHOTTING ################################ -# -# Save the DB on disk: -# -# save -# -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. -# -# In the example below the behaviour will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed -# -# Note: you can disable saving completely by commenting out all "save" lines. -# -# It is also possible to remove all the previously configured save -# points by adding a save directive with a single empty string argument -# like in the following example: -# -# save "" -save 900 1 -save 300 10 -save 60 10000 +# Save the DB to disk. +# +# save +# +# Redis will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# Snapshotting can be completely disabled with a single empty string argument +# as in following example: +# +# save "" +# +# Unless specified otherwise, by default Redis will save the DB: +# * After 3600 seconds (an hour) if at least 1 key changed +# * After 300 seconds (5 minutes) if at least 100 keys changed +# * After 60 seconds if at least 10000 keys changed +# +# You can set these explicitly by uncommenting the three following lines. +# +# save 3600 1 +# save 300 100 +# save 60 10000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. @@ -235,7 +398,7 @@ save 60 10000 stop-writes-on-bgsave-error yes # Compress string objects using LZF when dump .rdb databases? -# For default that's set to 'yes' as it's almost always a win. +# By default compression is enabled as it's almost always a win. # If you want to save some CPU in the saving child set it to 'no' but # the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes @@ -249,9 +412,37 @@ rdbcompression yes # tell the loading code to skip the check. rdbchecksum yes +# Enables or disables full sanitation checks for ziplist and listpack etc when +# loading an RDB or RESTORE payload. This reduces the chances of a assertion or +# crash later on while processing commands. +# Options: +# no - Never perform full sanitation +# yes - Always perform full sanitation +# clients - Perform full sanitation only for user connections. +# Excludes: RDB files, RESTORE commands received from the master +# connection, and client connections which have the +# skip-sanitize-payload ACL flag. +# The default should be 'clients' but since it currently affects cluster +# resharding via MIGRATE, it is temporarily set to 'no' by default. +# +# sanitize-dump-payload no + # The filename where to dump the DB dbfilename dump.rdb +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + # The working directory. # # The DB will be written inside this directory, with the filename specified @@ -310,11 +501,11 @@ dir /var/lib/keydb # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if replica-serve-stale-data is set to 'no' the replica will reply with -# an error "SYNC with master in progress" to all the kind of commands -# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, -# SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, -# COMMAND, POST, HOST: and LATENCY. +# 2) If replica-serve-stale-data is set to 'no' the replica will reply with +# an error "SYNC with master in progress" to all commands except: +# INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, +# UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, +# HOST and LATENCY. # replica-serve-stale-data yes @@ -377,9 +568,35 @@ repl-diskless-sync no # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if you know what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# received from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. # # repl-ping-replica-period 10 @@ -391,7 +608,8 @@ repl-diskless-sync-delay 5 # # It is important to make sure that this value is greater than the value # specified for repl-ping-replica-period otherwise a timeout will be detected -# every time there is low traffic between the master and the replica. +# every time there is low traffic between the master and the replica. The default +# value is 60 seconds. # # repl-timeout 60 @@ -416,21 +634,21 @@ repl-disable-tcp-nodelay no # resync is enough, just passing the portion of data the replica missed while # disconnected. # -# The bigger the replication backlog, the longer the time the replica can be -# disconnected and later be able to perform a partial resynchronization. +# The bigger the replication backlog, the longer the replica can endure the +# disconnect and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a replica connected. +# The backlog is only allocated if there is at least one replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected replicas for some time, the backlog -# will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last replica disconnected, for -# the backlog buffer to be freed. +# After a master has no connected replicas for some time, the backlog will be +# freed. The following option configures the amount of seconds that need to +# elapse, starting from the time the last replica disconnected, for the backlog +# buffer to be freed. # # Note that replicas never free the backlog for timeout, since they may be # promoted to masters later, and should be able to correctly "partially -# resynchronize" with the replicas: hence they should always accumulate backlog. +# resynchronize" with other replicas: hence they should always accumulate backlog. # # A value of 0 means to never release the backlog. # @@ -451,6 +669,18 @@ repl-disable-tcp-nodelay no # By default the priority is 100. replica-priority 100 +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes + # It is possible for a master to stop accepting writes if there are less than # N replicas connected, having a lag less or equal than M seconds. # @@ -480,8 +710,8 @@ replica-priority 100 # Another place where this info is available is in the output of the # "ROLE" command of a master. # -# The listed IP and address normally reported by a replica is obtained -# in the following way: +# The listed IP address and port normally reported by a replica is +# obtained in the following way: # # IP: The address is auto detected by checking the peer address # of the socket used by the replica to connect with the master. @@ -491,7 +721,7 @@ replica-priority 100 # listen for connections. # # However when port forwarding or Network Address Translation (NAT) is -# used, the replica may be actually reachable via different IP and port +# used, the replica may actually be reachable via different IP and port # pairs. The following two options can be used by a replica in order to # report to its master a specific set of IP and port, so that both INFO # and ROLE will report those values. @@ -502,9 +732,45 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 +############################### KEYS TRACKING ################################# + +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# a radix key indexed by key name, what clients have which keys. In turn +# this is used in order to send invalidation messages to clients. Please +# check this page to understand more about the feature: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum size is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. +# +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 + ################################## SECURITY ################################### -# Warning: since Redis is pretty fast an outside user can try up to +# Warning: since Redis is pretty fast, an outside user can try up to # 1 million passwords per second against a modern box. This means that you # should use very strong passwords, otherwise they will be very easy to break. # Note that because the password is really a shared secret between the client @@ -528,12 +794,14 @@ replica-priority 100 # AUTH (or the HELLO command AUTH option) in order to be authenticated and # start to work. # -# The ACL rules that describe what an user can do are the following: +# The ACL rules that describe what a user can do are the following: # # on Enable the user: it is possible to authenticate as this user. # off Disable the user: it's no longer possible to authenticate # with this user, however the already authenticated connections # will still work. +# skip-sanitize-payload RESTORE dump-payload sanitation is skipped. +# sanitize-payload RESTORE dump-payload is sanitized (default). # + Allow the execution of that command # - Disallow the execution of that command # +@ Allow the execution of all the commands in such category @@ -556,7 +824,12 @@ replica-priority 100 # It is possible to specify multiple patterns. # allkeys Alias for ~* # resetkeys Flush the list of allowed keys patterns. -# > Add this passowrd to the list of valid password for the user. +# & Add a glob-style pattern of Pub/Sub channels that can be +# accessed by the user. It is possible to specify multiple channel +# patterns. +# allchannels Alias for &* +# resetchannels Flush the list of allowed channel patterns. +# > Add this password to the list of valid password for the user. # For example >mypass will add "mypass" to the list. # This directive clears the "nopass" flag (see later). # < Remove this password from the list of valid passwords. @@ -598,26 +871,58 @@ replica-priority 100 # For more information about ACL configuration please refer to # the Redis web site at https://redis.io/topics/acl +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + # Using an external ACL file # # Instead of configuring users here in this file, it is possible to use # a stand-alone file just listing users. The two methods cannot be mixed: -# if you configure users here and at the same time you activate the exteranl +# if you configure users here and at the same time you activate the external # ACL file, the server will refuse to start. # # The format of the external ACL user file is exactly the same as the -# format that is used inside redis.conf to describe users. +# format that is used inside keydb.conf to describe users. # -# aclfile /etc/redis/users.acl +# aclfile /etc/keydb/users.acl -# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatiblity +# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility # layer on top of the new ACL system. The option effect will be just setting # the password for the default user. Clients will still authenticate using # AUTH as usually, or more explicitly with AUTH default # if they follow the new protocol: both will work. # +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. +# # requirepass foobared +# New users are initialized with restrictive permissions by default, via the +# equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it +# is possible to manage access to Pub/Sub channels with ACL rules as well. The +# default Pub/Sub channels permission if new users is controlled by the +# acl-pubsub-default configuration directive, which accepts one of these values: +# +# allchannels: grants access to all Pub/Sub channels +# resetchannels: revokes access to all Pub/Sub channels +# +# To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default +# defaults to the 'allchannels' permission. +# +# Future compatibility note: it is very likely that in a future version of Redis +# the directive's default of 'allchannels' will be changed to 'resetchannels' in +# order to provide better out-of-the-box Pub/Sub security. Therefore, it is +# recommended that you explicitly define Pub/Sub permissions for all users +# rather then rely on implicit default values. Once you've set explicit +# Pub/Sub for all existing users, you should uncomment the following line. +# +# acl-pubsub-default resetchannels + # Command renaming (DEPRECATED). # # ------------------------------------------------------------------------ @@ -654,6 +959,11 @@ replica-priority 100 # Once the limit is reached Redis will close all the new connections sending # an error 'max number of clients reached'. # +# IMPORTANT: When Redis Cluster is used, the max number of connections is also +# shared with the cluster bus: every node in the cluster will use two +# connections, one incoming and another outgoing. It is important to size the +# limit accordingly in case of very large clusters. +# # maxclients 10000 ############################## MEMORY MANAGEMENT ################################ @@ -701,14 +1011,12 @@ replica-priority 100 # Both LRU, LFU and volatile-ttl are implemented using approximated # randomized algorithms. # -# Note: with any of the above policies, Redis will return an error on write -# operations, when there are no suitable keys for eviction. -# -# At the date of writing these commands are: set setnx setex append -# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd -# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby -# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby -# getset mset msetnx exec sort +# Note: with any of the above policies, when there are no suitable keys for +# eviction, Redis will return an error on write operations that require +# more memory. These are usually commands that create new keys, add data or +# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, +# SORT (due to the STORE argument), and EXEC (if the transaction includes any +# command that requires memory). # # The default is: # @@ -716,8 +1024,8 @@ replica-priority 100 # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated # algorithms (in order to save memory), so you can tune it for speed or -# accuracy. For default Redis will check five keys and pick the one that was -# used less recently, you can change the sample size using the following +# accuracy. By default Redis will check five keys and pick the one that was +# used least recently, you can change the sample size using the following # configuration directive. # # The default of 5 produces good enough results. 10 Approximates very closely @@ -725,6 +1033,14 @@ replica-priority 100 # # maxmemory-samples 5 +# Eviction processing is designed to function well with the default setting. +# If there is an unusually large amount of write traffic, this value may need to +# be increased. Decreasing this value may reduce latency at the risk of +# eviction processing effectiveness +# 0 = minimum latency, 10 = default, 100 = process without regard to latency +# +# maxmemory-eviction-tenacity 10 + # Starting from Redis 5, by default a replica will ignore its maxmemory setting # (unless it is promoted to master after a failover or manually). It means # that the eviction of keys will be just handled by the master, sending the @@ -745,6 +1061,23 @@ replica-priority 100 # # replica-ignore-maxmemory yes +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tolerate less already expired keys still present +# in the system. It's a tradeoff between memory, CPU and latency. +# +# active-expire-effort 1 + ############################# LAZY FREEING #################################### # Redis has two primitives to delete keys. One is called DEL and is a blocking @@ -787,13 +1120,117 @@ replica-priority 100 # In all the above cases the default is to delete objects in a blocking way, # like if DEL was called. However you can configure each case specifically # in order to instead release memory in a non-blocking way like if UNLINK -# was called, using the following configuration directives: +# was called, using the following configuration directives. lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + +# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous +# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the +# commands. When neither flag is passed, this directive will be used to determine +# if the data should be deleted asynchronously. + +lazyfree-lazy-user-flush no + +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speed up the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usual. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using keydb-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis threads, otherwise you'll not +# be able to notice the improvements. + +############################ KERNEL OOM CONTROL ############################## + +# On Linux, it is possible to hint the kernel OOM killer on what processes +# should be killed first when out of memory. +# +# Enabling this feature makes Redis actively control the oom_score_adj value +# for all its processes, depending on their role. The default scores will +# attempt to have background child processes killed before all others, and +# replicas killed before masters. +# +# Redis supports three options: +# +# no: Don't make changes to oom-score-adj (default). +# yes: Alias to "relative" see below. +# absolute: Values in oom-score-adj-values are written as is to the kernel. +# relative: Values are used relative to the initial value of oom_score_adj when +# the server starts and are then clamped to a range of -1000 to 1000. +# Because typically the initial value is 0, they will often match the +# absolute values. +oom-score-adj no + +# When oom-score-adj is used, this directive controls the specific values used +# for master, replica and background child processes. Values range -2000 to +# 2000 (higher means more likely to be killed). +# +# Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) +# can freely increase their value, but not decrease it below its initial +# settings. This means that setting oom-score-adj to "relative" and setting the +# oom-score-adj-values to positive values will always succeed. +oom-score-adj-values 0 200 800 + + +#################### KERNEL transparent hugepage CONTROL ###################### + +# Usually the kernel Transparent Huge Pages control is set to "madvise" or +# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which +# case this config has no effect. On systems in which it is set to "always", +# keydb will attempt to disable it specifically for the keydb process in order +# to avoid latency problems specifically with fork(2) and CoW. +# If for some reason you prefer to keep it enabled, you can set this config to +# "no" and the kernel global to "always". + +disable-thp yes + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is @@ -812,7 +1249,7 @@ replica-lazy-flush no # If the AOF is enabled on startup Redis will load the AOF, that is the file # with the better durability guarantees. # -# Please check http://redis.io/topics/persistence for more information. +# Please check https://redis.io/topics/persistence for more information. appendonly no @@ -918,8 +1355,8 @@ aof-load-truncated yes # # [RDB file][AOF tail] # -# When loading Redis recognizes that the AOF file starts with the "REDIS" -# string and loads the prefixed RDB file, and continues loading the AOF +# When loading, Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, then continues loading the AOF # tail. aof-use-rdb-preamble yes @@ -933,7 +1370,7 @@ aof-use-rdb-preamble yes # # When a long running script exceeds the maximum execution time only the # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be -# used to stop a script that did not yet called write commands. The second +# used to stop a script that did not yet call any write commands. The second # is the only way to shut down the server in the case a write command was # already issued by the script but the user doesn't want to wait for the natural # termination of the script. @@ -965,7 +1402,7 @@ lua-time-limit 5000 # Cluster node timeout is the amount of milliseconds a node must be unreachable # for it to be considered in failure state. -# Most other internal time limits are multiple of the node timeout. +# Most other internal time limits are a multiple of the node timeout. # # cluster-node-timeout 15000 @@ -992,18 +1429,18 @@ lua-time-limit 5000 # the failover if, since the last interaction with the master, the time # elapsed is greater than: # -# (node-timeout * replica-validity-factor) + repl-ping-replica-period +# (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period # -# So for example if node-timeout is 30 seconds, and the replica-validity-factor +# So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the # replica will not try to failover if it was not able to talk with the master # for longer than 310 seconds. # -# A large replica-validity-factor may allow replicas with too old data to failover +# A large cluster-replica-validity-factor may allow replicas with too old data to failover # a master, while a too small value may prevent the cluster from being able to # elect a replica at all. # -# For maximum availability, it is possible to set the replica-validity-factor +# For maximum availability, it is possible to set the cluster-replica-validity-factor # to a value of 0, which means, that replicas will always try to failover the # master regardless of the last time they interacted with the master. # (However they'll always try to apply a delay proportional to their @@ -1027,14 +1464,23 @@ lua-time-limit 5000 # master in your cluster. # # Default is 1 (replicas migrate only if their masters remain with at least -# one replica). To disable migration just set it to a very large value. +# one replica). To disable migration just set it to a very large value or +# set cluster-allow-replica-migration to 'no'. # A value of 0 can be set but is useful only for debugging and dangerous # in production. # # cluster-migration-barrier 1 +# Turning off this option allows to use less automatic cluster configuration. +# It both disables migration to orphaned masters and migration from masters +# that became empty. +# +# Default is 'yes' (allow automatic migrations). +# +# cluster-allow-replica-migration yes + # By default Redis Cluster nodes stop accepting queries if they detect there -# is at least an hash slot uncovered (no available node is serving it). +# is at least a hash slot uncovered (no available node is serving it). # This way if the cluster is partially down (for example a range of hash slots # are no longer covered) all the cluster becomes, eventually, unavailable. # It automatically returns available as soon as all the slots are covered again. @@ -1047,7 +1493,7 @@ lua-time-limit 5000 # cluster-require-full-coverage yes # This option, when set to yes, prevents replicas from trying to failover its -# master during master failures. However the master can still perform a +# master during master failures. However the replica can still perform a # manual failover, if forced to do so. # # This is useful in different scenarios, especially in the case of multiple @@ -1057,7 +1503,7 @@ lua-time-limit 5000 # cluster-replica-no-failover no # In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. +# available at https://redis.io web site. ########################## CLUSTER DOCKER/NAT support ######################## @@ -1067,16 +1513,21 @@ lua-time-limit 5000 # # In order to make Redis Cluster working in such environments, a static # configuration where each node knows its public address is needed. The -# following two options are used for this scope, and are: +# following four options are used for this scope, and are: # # * cluster-announce-ip # * cluster-announce-port +# * cluster-announce-tls-port # * cluster-announce-bus-port # -# Each instruct the node about its address, client port, and cluster message -# bus port. The information is then published in the header of the bus packets -# so that other nodes will be able to correctly map the address of the node -# publishing the information. +# Each instructs the node about its address, client ports (for connections +# without and with TLS) and cluster message bus port. The information is then +# published in the header of the bus packets so that other nodes will be able to +# correctly map the address of the node publishing the information. +# +# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set +# to zero, then cluster-announce-port refers to the TLS port. Note also that +# cluster-announce-tls-port has no effect if cluster-tls is set to no. # # If the above options are not used, the normal Redis Cluster auto-detection # will be used instead. @@ -1084,12 +1535,13 @@ lua-time-limit 5000 # Note that when remapped, the bus port may not be at the fixed offset of # clients port + 10000, so you can specify any port and bus-port depending # on how they get remapped. If the bus-port is not set, a fixed offset of -# 10000 will be used as usually. +# 10000 will be used as usual. # # Example: # # cluster-announce-ip 10.1.1.5 -# cluster-announce-port 6379 +# cluster-announce-tls-port 6379 +# cluster-announce-port 0 # cluster-announce-bus-port 6380 ################################## SLOW LOG ################################### @@ -1140,7 +1592,7 @@ latency-monitor-threshold 0 ############################# EVENT NOTIFICATION ############################## # Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/notifications +# This feature is documented at https://redis.io/topics/notifications # # For instance if keyspace events notification is enabled, and a client # performs a DEL operation on key "foo" stored in the Database 0, two @@ -1162,7 +1614,12 @@ latency-monitor-threshold 0 # z Sorted set commands # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# t Stream commands +# d Module key type events +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxetd, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications @@ -1209,7 +1666,7 @@ notify-keyspace-events "" # two kind of inline requests that were anyway illegal: an empty request # or any request that starts with "/" (there are no Redis commands starting # with such a slash). Normal RESP2/RESP3 requests are completely out of the -# path of the Gopher protocol implementation and are served as usually as well. +# path of the Gopher protocol implementation and are served as usual as well. # # If you open a connection to Redis when Gopher is enabled and send it # a string like "/foo", if there is a key named "/foo" it is served via the @@ -1233,8 +1690,11 @@ notify-keyspace-events "" # # So use the 'requirepass' option to protect your instance. # -# To enable Gopher support uncomment the following line and set -# the option from no (the default) to yes. +# Note that Gopher is not currently supported when 'io-threads-do-reads' +# is enabled. +# +# To enable Gopher support, uncomment the following line and set the option +# from no (the default) to yes. # # gopher-enabled no @@ -1310,7 +1770,7 @@ hll-sparse-max-bytes 3000 # maximum number of items it may contain before switching to a new node when # appending new stream entries. If any of the following settings are set to # zero, the limit is ignored, so for instance it is possible to set just a -# max entires limit by setting max-bytes to 0 and max-entries to the desired +# max entries limit by setting max-bytes to 0 and max-entries to the desired # value. stream-node-max-bytes 4096 stream-node-max-entries 100 @@ -1381,8 +1841,8 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # client-query-buffer-limit 1gb # In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited ot 512 mb. However you can change this limit -# here. +# strings, are normally limited to 512 mb. However you can change this limit +# here, but must be 1mb or greater # # proto-max-bulk-len 512mb @@ -1410,9 +1870,9 @@ hz 10 # # Since the default HZ value by default is conservatively set to 10, Redis # offers, and enables by default, the ability to use an adaptive HZ value -# which will temporary raise when there are many connected clients. +# which will temporarily raise when there are many connected clients. # -# When dynamic HZ is enabled, the actual configured HZ will be used as +# When dynamic HZ is enabled, the actual configured HZ will be used # as a baseline, but multiples of the configured HZ value will be actually # used as needed once more clients are connected. In this way an idle # instance will use very little CPU time while a busy instance will be @@ -1425,7 +1885,7 @@ dynamic-hz yes # big latency spikes. aof-rewrite-incremental-fsync yes -# When redis saves RDB file, if the following option is enabled +# When KeyDB saves RDB file, if the following option is enabled # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid # big latency spikes. @@ -1477,7 +1937,7 @@ rdb-save-incremental-fsync yes # for the key counter to be divided by two (or decremented if it has a value # less <= 10). # -# The default value for the lfu-decay-time is 1. A Special value of 0 means to +# The default value for the lfu-decay-time is 1. A special value of 0 means to # decay the counter every time it happens to be scanned. # # lfu-log-factor 10 @@ -1501,7 +1961,7 @@ rdb-save-incremental-fsync yes # restart is needed in order to lower the fragmentation, or at least to flush # away all the data and create it again. However thanks to this feature # implemented by Oran Agra for Redis 4.0 this process can happen at runtime -# in an "hot" way, while the server is running. +# in a "hot" way, while the server is running. # # Basically when the fragmentation is over a certain level (see the # configuration options below) Redis will start to create new copies of the @@ -1549,6 +2009,42 @@ rdb-save-incremental-fsync yes # the main dictionary scan # active-defrag-max-scan-fields 1000 +# Jemalloc background thread for purging will be enabled by default +jemalloc-bg-thread yes + +# It is possible to pin different threads and processes of Redis to specific +# CPUs in your system, in order to maximize the performances of the server. +# This is useful both in order to pin different Redis threads in different +# CPUs, but also in order to make sure that multiple Redis instances running +# in the same host will be pinned to different CPUs. +# +# Normally you can do this using the "taskset" command, however it is also +# possible to this via Redis configuration directly, both in Linux and FreeBSD. +# +# You can pin the server/IO threads, bio threads, aof rewrite child process, and +# the bgsave child process. The syntax to specify the cpu list is the same as +# the taskset command: +# +# Set KeyDB server/io threads to cpu affinity 0,2,4,6: +# server_cpulist 0-7:2 +# +# Set bio threads to cpu affinity 1,3: +# bio_cpulist 1,3 +# +# Set aof rewrite child process to cpu affinity 8,9,10,11: +# aof_rewrite_cpulist 8-11 +# +# Set bgsave child process to cpu affinity 1,10,11 +# bgsave_cpulist 1,10-11 + +# In some cases KeyDB will emit warnings and even refuse to start if it detects +# that the system is in bad state, it is possible to suppress these warnings +# by setting the following config which takes a space delimited list of warnings +# to suppress +# +# ignore-warnings ARM64-COW-BUG + + # Path to directory for file backed scratchpad. The file backed scratchpad # reduces memory requirements by storing rarely accessed data on disk # instead of RAM. A temporary file will be created in this directory. @@ -1574,3 +2070,4 @@ server-threads 2 # Enable FLASH support? (Enterprise Only) # storage-provider flash /path/to/flash/db + diff --git a/runtest b/runtest index ade1bd09a..c6349d118 100755 --- a/runtest +++ b/runtest @@ -2,6 +2,8 @@ TCL_VERSIONS="8.5 8.6" TCLSH="" +export ASAN_OPTIONS=allocator_may_return_null=1 $ASAN_OPTIONS + for VERSION in $TCL_VERSIONS; do TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL done diff --git a/runtest-cluster b/runtest-cluster index 27829a5fe..a86e93141 100755 --- a/runtest-cluster +++ b/runtest-cluster @@ -8,7 +8,7 @@ done if [ -z $TCLSH ] then - echo "You need tcl 8.5 or newer in order to run the Redis Sentinel test" + echo "You need tcl 8.5 or newer in order to run the Redis Cluster test" exit 1 fi $TCLSH tests/cluster/run.tcl $* diff --git a/runtest-moduleapi b/runtest-moduleapi index 268506160..2e5bffcba 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -1,6 +1,7 @@ #!/bin/sh TCL_VERSIONS="8.5 8.6" TCLSH="" +[ -z "$MAKE" ] && MAKE=make for VERSION in $TCL_VERSIONS; do TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL @@ -8,11 +9,11 @@ done if [ -z $TCLSH ] then - echo "You need tcl 8.5 or newer in order to run the Redis test" + echo "You need tcl 8.5 or newer in order to run the Redis ModuleApi test" exit 1 fi -make -C tests/modules && \ +$MAKE -C tests/modules && \ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/commandfilter \ --single unit/moduleapi/fork \ @@ -22,6 +23,7 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/hooks \ --single unit/moduleapi/misc \ --single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/blockonbackground \ --single unit/moduleapi/scan \ --single unit/moduleapi/datatype \ --single unit/moduleapi/auth \ @@ -29,4 +31,11 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/blockedclient \ --single unit/moduleapi/moduleloadsave \ --single unit/moduleapi/getkeys \ +--single unit/moduleapi/timers \ +--single unit/moduleapi/test_lazyfree \ +--single unit/moduleapi/defrag \ +--single unit/moduleapi/hash \ +--single unit/moduleapi/zset \ +--single unit/moduleapi/stream \ +--config server-threads 3 \ "${@}" diff --git a/sentinel.conf b/sentinel.conf index 2ec6717d5..5ab5a8b5c 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -124,6 +124,42 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # Default is 30 seconds. sentinel down-after-milliseconds mymaster 30000 +# IMPORTANT NOTE: starting with KeyDB 6.2 ACL capability is supported for +# Sentinel mode, please refer to the KeyDB website https://redis.io/topics/acl +# for more details. + +# Sentinel's ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@admin +@connection ~* on >ffa9203c493aa99 +# +# For more information about ACL configuration please refer to the Redis +# website at https://redis.io/topics/acl and redis server configuration +# template redis.conf. + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the external +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/sentinel-users.acl + # requirepass # # You can configure Sentinel itself to require a password, however when doing @@ -131,6 +167,29 @@ sentinel down-after-milliseconds mymaster 30000 # other Sentinels. So you need to configure all your Sentinels in a given # group with the same "requirepass" password. Check the following documentation # for more info: https://redis.io/topics/sentinel +# +# IMPORTANT NOTE: starting with Redis 6.2 "requirepass" is a compatibility +# layer on top of the ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. +# +# New config files are advised to use separate authentication control for +# incoming connections (via ACL), and for outgoing connections (via +# sentinel-user and sentinel-pass) +# +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. + +# sentinel sentinel-user +# +# You can configure Sentinel to authenticate with other Sentinels with specific +# user name. + +# sentinel sentinel-pass +# +# The password for Sentinel to authenticate with other Sentinels. If sentinel-user +# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate. # sentinel parallel-syncs # @@ -262,3 +321,21 @@ sentinel deny-scripts-reconfig yes # is possible to just rename a command to itself: # # SENTINEL rename-command mymaster CONFIG CONFIG + +# HOSTNAMES SUPPORT +# +# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR +# to specify an IP address. Also, it requires the Redis replica-announce-ip +# keyword to specify only IP addresses. +# +# You may enable hostnames support by enabling resolve-hostnames. Note +# that you must make sure your DNS is configured properly and that DNS +# resolution does not introduce very long delays. +# +SENTINEL resolve-hostnames no + +# When resolve-hostnames is enabled, Sentinel still uses IP addresses +# when exposing instances to users, configuration files, etc. If you want +# to retain the hostnames when announced, enable announce-hostnames below. +# +SENTINEL announce-hostnames no diff --git a/src/Makefile b/src/Makefile index 966ce4400..92bb346f4 100644 --- a/src/Makefile +++ b/src/Makefile @@ -15,14 +15,16 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') -OPTIMIZATION?=-O2 -flto -DEPENDENCY_TARGETS=hiredis linenoise lua rocksdb +OPTIMIZATION?=-O2 +DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram rocksdb NODEPS:=clean distclean # Default settings -STD=-std=c11 -pedantic -DREDIS_STATIC='' +STD=-pedantic -DREDIS_STATIC='' CXX_STD=-std=c++14 -pedantic -fno-rtti -D__STDC_FORMAT_MACROS ifneq (,$(findstring clang,$(CC))) + STD+=-Wno-c11-extensions +else ifneq (,$(findstring FreeBSD,$(uname_S))) STD+=-Wno-c11-extensions endif @@ -30,6 +32,16 @@ endif WARN=-Wall -W -Wno-missing-field-initializers OPT=$(OPTIMIZATION) +# Detect if the compiler supports C11 _Atomic +C11_ATOMIC := $(shell sh -c 'echo "\#include " > foo.c; \ + $(CC) -std=c11 -c foo.c -o foo.o > /dev/null 2>&1; \ + if [ -f foo.o ]; then echo "yes"; rm foo.o; fi; rm foo.c') +ifeq ($(C11_ATOMIC),yes) + STD+=-std=c11 +else + STD+=-std=c99 +endif + PREFIX?=/usr/local INSTALL_BIN=$(PREFIX)/bin INSTALL=install @@ -132,10 +144,12 @@ endif ifeq ($(uname_S),SunOS) # SunOS - ifneq ($(@@),32bit) - CFLAGS+= -m64 + ifeq ($(findstring -m32,$(FINAL_CFLAGS)),) + CFLAGS+=-m64 CXXFLAGS+= -m64 - LDFLAGS+= -m64 + endif + ifeq ($(findstring -m32,$(FINAL_LDFLAGS)),) + LDFLAGS+=-m64 endif DEBUG=-g DEBUG_FLAGS=-g @@ -148,8 +162,18 @@ else ifeq ($(uname_S),Darwin) # Darwin FINAL_LIBS+= -ldl + # Homebrew's OpenSSL is not linked to /usr/local to avoid + # conflicts with the system's LibreSSL installation so it + # must be referenced explicitly during build. +ifeq ($(uname_M),arm64) + # Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX + OPENSSL_CFLAGS=-I/opt/homebrew/opt/openssl/include + OPENSSL_LDFLAGS=-L/opt/homebrew/opt/openssl/lib +else + # Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib +endif else ifeq ($(uname_S),AIX) # AIX @@ -217,11 +241,13 @@ endif endif endif # Include paths to dependencies -FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/license/ -FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/rocksdb/include/ -I../deps/license +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/license/ +FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/rocksdb/include/ -I../deps/license # Determine systemd support and/or build preference (defaulting to auto-detection) BUILD_WITH_SYSTEMD=no +LIBSYSTEMD_LIBS=-lsystemd + # If 'USE_SYSTEMD' in the environment is neither "no" nor "yes", try to # auto-detect libsystemd's presence and link accordingly. ifneq ($(USE_SYSTEMD),no) @@ -230,17 +256,18 @@ ifneq ($(USE_SYSTEMD),no) # (unless a later check tells us otherwise) ifeq ($(LIBSYSTEMD_PKGCONFIG),0) BUILD_WITH_SYSTEMD=yes + LIBSYSTEMD_LIBS=$(shell $(PKG_CONFIG) --libs libsystemd) endif endif + +# If 'USE_SYSTEMD' is set to "yes" use pkg-config if available or fall back to +# default -lsystemd. ifeq ($(USE_SYSTEMD),yes) -ifneq ($(LIBSYSTEMD_PKGCONFIG),0) -$(error USE_SYSTEMD is set to "$(USE_SYSTEMD)", but $(PKG_CONFIG) cannot find libsystemd) -endif -# Force building with libsystemd BUILD_WITH_SYSTEMD=yes endif + ifeq ($(BUILD_WITH_SYSTEMD),yes) - FINAL_LIBS+=$(shell $(PKG_CONFIG) --libs libsystemd) + FINAL_LIBS+=$(LIBSYSTEMD_LIBS) FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD endif @@ -271,10 +298,9 @@ ifeq ($(MALLOC),memkind) endif ifeq ($(BUILD_TLS),yes) - FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CXXFLAGS) - FINAL_CXXFLAGS+=-DUSE_OPENSSL $(OPENSSL_CXXFLAGS) - FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) - FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto + FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS) + FINAL_CXXFLAGS+=-DUSE_OPENSSL $(OPENSSL_CXXFLAGS) + FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?) ifeq ($(LIBSSL_PKGCONFIG),0) LIBSSL_LIBS=$(shell $(PKG_CONFIG) --libs libssl) @@ -290,6 +316,17 @@ endif FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS) endif +ifndef V + define MAKE_INSTALL + @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$(1)$(ENDCOLOR) 1>&2 + @$(INSTALL) $(1) $(2) + endef +else + define MAKE_INSTALL + $(INSTALL) $(1) $(2) + endef +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_CXX=$(QUIET_CC)$(CXX) $(FINAL_CXXFLAGS) KEYDB_AS=$(QUIET_CC) as --64 -g @@ -312,11 +349,11 @@ endif REDIS_SERVER_NAME=keydb-server$(PROG_SUFFIX) REDIS_SENTINEL_NAME=keydb-sentinel$(PROG_SUFFIX) -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/rocksdb.o storage/rocksdbfactory.o storage/teststorageprovider.o keydbutils.o StorageCache.o $(ASM_OBJ) +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/rocksdb.o storage/rocksdbfactory.o storage/teststorageprovider.o keydbutils.o StorageCache.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ) REDIS_CLI_NAME=keydb-cli$(PROG_SUFFIX) -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o motd.o $(ASM_OBJ) +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o motd.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ) REDIS_BENCHMARK_NAME=keydb-benchmark$(PROG_SUFFIX) -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o redis-benchmark.o storage-lite.o fastlock.o new.o $(ASM_OBJ) +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o redis-benchmark.o storage-lite.o fastlock.o new.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ) REDIS_CHECK_RDB_NAME=keydb-check-rdb$(PROG_SUFFIX) REDIS_CHECK_AOF_NAME=keydb-check-aof$(PROG_SUFFIX) @@ -390,10 +427,7 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) # keydb-benchmark $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS) - -dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c - $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS) DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) -include $(DEP) @@ -411,7 +445,7 @@ DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ $(KEYDB_AS) $< -o $@ clean: - rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov KeyDB.info lcov-html Makefile.dep dict-benchmark + rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov KeyDB.info lcov-html Makefile.dep rm -rf storage/*.o rm -rf keydb-server rm -f $(DEP) @@ -426,10 +460,10 @@ distclean: clean .PHONY: distclean -test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) +test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) @(cd ..; ./runtest) -test-sentinel: $(REDIS_SENTINEL_NAME) +test-sentinel: $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) @(cd ..; ./runtest-sentinel) check: test @@ -441,10 +475,6 @@ lcov: @genhtml --legend -o lcov-html KeyDB.info @genhtml --legend -o lcov-html KeyDB.info | grep lines | awk '{print $$2;}' | sed 's/%//g' -test-sds: sds.c sds.h - $(REDIS_CC) sds.c zmalloc.cpp -DSDS_TEST_MAIN $(FINAL_LIBS) -o /tmp/sds_test - /tmp/sds_test - .PHONY: lcov bench: $(REDIS_BENCHMARK_NAME) @@ -466,18 +496,18 @@ valgrind: $(MAKE) OPTIMIZATION="-O0" USEASM="false" MALLOC="libc" CFLAGS="-DSANITIZE" CXXFLAGS="-DSANITIZE" helgrind: - $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" + $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" REDIS_CFLAGS="-I/usr/local/include" REDIS_LDFLAGS="-L/usr/local/lib" src/help.h: @../utils/generate-command-help.rb > help.h install: all @mkdir -p $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CHECK_RDB_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN) + $(call MAKE_INSTALL,$(REDIS_SERVER_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_BENCHMARK_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_CLI_NAME),$(INSTALL_BIN)) + @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_RDB_NAME) + @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_AOF_NAME) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME) uninstall: diff --git a/src/acl.cpp b/src/acl.cpp index 7f8ab74bd..654a9d6a2 100644 --- a/src/acl.cpp +++ b/src/acl.cpp @@ -55,6 +55,10 @@ list *UsersToLoad; /* This is a list of users found in the configuration file list *ACLLog; /* Our security log, the user is able to inspect that using the ACL LOG command .*/ +static rax *commandId = NULL; /* Command name to id mapping */ + +static unsigned long nextid = 0; /* Next command id that has not been assigned */ + struct ACLCategoryItem { const char *name; uint64_t flag; @@ -88,18 +92,22 @@ struct ACLUserFlag { const char *name; uint64_t flag; } ACLUserFlags[] = { + /* Note: the order here dictates the emitted order at ACLDescribeUser */ {"on", USER_FLAG_ENABLED}, {"off", USER_FLAG_DISABLED}, {"allkeys", USER_FLAG_ALLKEYS}, + {"allchannels", USER_FLAG_ALLCHANNELS}, {"allcommands", USER_FLAG_ALLCOMMANDS}, {"nopass", USER_FLAG_NOPASS}, + {"skip-sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD_SKIP}, + {"sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD}, {NULL,0} /* Terminator. */ }; void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); -void ACLFreeLogEntry(struct ACLLogEntry *le); +void ACLFreeLogEntry(const void *le); /* The length of the string representation of a hashed password. */ #define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 @@ -169,15 +177,15 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) { return sdsnewlen(hex,HASH_PASSWORD_LEN); } -/* Given a hash and the hash length, returns C_OK if it is a valid password +/* Given a hash and the hash length, returns C_OK if it is a valid password * hash, or C_ERR otherwise. */ int ACLCheckPasswordHash(unsigned char *hash, int hashlen) { if (hashlen != HASH_PASSWORD_LEN) { - return C_ERR; + return C_ERR; } - + /* Password hashes can only be characters that represent - * hexadecimal values, which are numbers and lowercase + * hexadecimal values, which are numbers and lowercase * characters 'a' through 'f'. */ for(int i = 0; i < HASH_PASSWORD_LEN; i++) { char c = hash[i]; @@ -240,16 +248,20 @@ user *ACLCreateUser(const char *name, size_t namelen) { if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL; user *u = (user*)zmalloc(sizeof(*u), MALLOC_LOCAL); u->name = sdsnewlen(name,namelen); - u->flags = USER_FLAG_DISABLED; + u->flags = USER_FLAG_DISABLED | g_pserver->acl_pubsub_default; u->allowed_subcommands = NULL; u->passwords = listCreate(); u->patterns = listCreate(); + u->channels = listCreate(); listSetMatchMethod(u->passwords,ACLListMatchSds); listSetFreeMethod(u->passwords,ACLListFreeSds); listSetDupMethod(u->passwords,ACLListDupSds); listSetMatchMethod(u->patterns,ACLListMatchSds); listSetFreeMethod(u->patterns,ACLListFreeSds); listSetDupMethod(u->patterns,ACLListDupSds); + listSetMatchMethod(u->channels,ACLListMatchSds); + listSetFreeMethod(u->channels,ACLListFreeSds); + listSetDupMethod(u->channels,ACLListDupSds); memset(u->allowed_commands,0,sizeof(u->allowed_commands)); raxInsert(Users,(unsigned char*)name,namelen,u,NULL); return u; @@ -278,6 +290,7 @@ void ACLFreeUser(user *u) { sdsfree(u->name); listRelease(u->passwords); listRelease(u->patterns); + listRelease(u->channels); ACLResetSubcommands(u); zfree(u); } @@ -291,14 +304,14 @@ void ACLFreeUserAndKillClients(user *u) { listRewind(g_pserver->clients,&li); while ((ln = listNext(&li)) != NULL) { client *c = (client*)listNodeValue(ln); - if (c->puser == u) { - /* We'll free the conenction asynchronously, so + if (c->user == u) { + /* We'll free the connection asynchronously, so * in theory to set a different user is not needed. * However if there are bugs in Redis, soon or later * this may result in some security hole: it's much * more defensive to set the default user and put * it in non authenticated mode. */ - c->puser = DefaultUser; + c->user = DefaultUser; c->authenticated = 0; /* We will write replies to this client later, so we can't * close it directly even if async. */ @@ -318,8 +331,10 @@ void ACLFreeUserAndKillClients(user *u) { void ACLCopyUser(user *dst, user *src) { listRelease(dst->passwords); listRelease(dst->patterns); + listRelease(dst->channels); dst->passwords = listDup(src->passwords); dst->patterns = listDup(src->patterns); + dst->channels = listDup(src->channels); memcpy(dst->allowed_commands,src->allowed_commands, sizeof(dst->allowed_commands)); dst->flags = src->flags; @@ -601,9 +616,10 @@ sds ACLDescribeUser(user *u) { /* Flags. */ for (int j = 0; ACLUserFlags[j].flag; j++) { - /* Skip the allcommands and allkeys flags because they'll be emitted - * later as ~* and +@all. */ + /* Skip the allcommands, allkeys and allchannels flags because they'll + * be emitted later as +@all, ~* and &*. */ if (ACLUserFlags[j].flag == USER_FLAG_ALLKEYS || + ACLUserFlags[j].flag == USER_FLAG_ALLCHANNELS || ACLUserFlags[j].flag == USER_FLAG_ALLCOMMANDS) continue; if (u->flags & ACLUserFlags[j].flag) { res = sdscat(res,ACLUserFlags[j].name); @@ -635,6 +651,20 @@ sds ACLDescribeUser(user *u) { } } + /* Pub/sub channel patterns. */ + if (u->flags & USER_FLAG_ALLCHANNELS) { + res = sdscatlen(res,"&* ",3); + } else { + res = sdscatlen(res,"resetchannels ",14); + listRewind(u->channels,&li); + while((ln = listNext(&li))) { + sds thispat = (sds)listNodeValue(ln); + res = sdscatlen(res,"&",1); + res = sdscatsds(res,thispat); + res = sdscatlen(res," ",1); + } + } + /* Command rules. */ sds rules = ACLDescribeUserCommandRules(u); res = sdscatsds(res,rules); @@ -680,7 +710,6 @@ void ACLResetSubcommands(user *u) { u->allowed_subcommands = NULL; } - /* Add a subcommand to the list of subcommands for the user 'u' and * the command id specified. */ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { @@ -741,6 +770,12 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * It is possible to specify multiple patterns. * allkeys Alias for ~* * resetkeys Flush the list of allowed keys patterns. + * & Add a pattern of channels that can be mentioned as part of + * Pub/Sub commands. For instance &* allows all the channels. The + * pattern is a glob-style pattern like the one of PSUBSCRIBE. + * It is possible to specify multiple patterns. + * allchannels Alias for &* + * resetchannels Flush the list of allowed keys patterns. * > Add this password to the list of valid password for the user. * For example >mypass will add "mypass" to the list. * This directive clears the "nopass" flag (see later). @@ -779,14 +814,14 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * * When an error is returned, errno is set to the following values: * - * EINVAL: The specified opcode is not understood or the key pattern is + * EINVAL: The specified opcode is not understood or the key/channel pattern is * invalid (contains non allowed characters). * ENOENT: The command name or command category provided with + or - is not * known. - * EBUSY: The subcommand you want to add is about a command that is currently - * fully added. * EEXIST: You are adding a key pattern after "*" was already added. This is * almost surely an error on the user side. + * EISDIR: You are adding a channel pattern after "*" was already added. This is + * almost surely an error on the user side. * ENODEV: The password you are trying to remove from the user does not exist. * EBADMSG: The hash you are trying to add is not a valid hash. */ @@ -799,6 +834,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"off")) { u->flags |= USER_FLAG_DISABLED; u->flags &= ~USER_FLAG_ENABLED; + } else if (!strcasecmp(op,"skip-sanitize-payload")) { + u->flags |= USER_FLAG_SANITIZE_PAYLOAD_SKIP; + u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD; + } else if (!strcasecmp(op,"sanitize-payload")) { + u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD_SKIP; + u->flags |= USER_FLAG_SANITIZE_PAYLOAD; } else if (!strcasecmp(op,"allkeys") || !strcasecmp(op,"~*")) { @@ -807,6 +848,14 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"resetkeys")) { u->flags &= ~USER_FLAG_ALLKEYS; listEmpty(u->patterns); + } else if (!strcasecmp(op,"allchannels") || + !strcasecmp(op,"&*")) + { + u->flags |= USER_FLAG_ALLCHANNELS; + listEmpty(u->channels); + } else if (!strcasecmp(op,"resetchannels")) { + u->flags &= ~USER_FLAG_ALLCHANNELS; + listEmpty(u->channels); } else if (!strcasecmp(op,"allcommands") || !strcasecmp(op,"+@all")) { @@ -874,12 +923,29 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); - /* Avoid re-adding the same pattern multiple times. */ + /* Avoid re-adding the same key pattern multiple times. */ if (ln == NULL) listAddNodeTail(u->patterns,newpat); else sdsfree(newpat); u->flags &= ~USER_FLAG_ALLKEYS; + } else if (op[0] == '&') { + if (u->flags & USER_FLAG_ALLCHANNELS) { + errno = EISDIR; + return C_ERR; + } + if (ACLStringHasSpaces(op+1,oplen-1)) { + errno = EINVAL; + return C_ERR; + } + sds newpat = sdsnewlen(op+1,oplen-1); + listNode *ln = listSearchKey(u->channels,newpat); + /* Avoid re-adding the same channel pattern multiple times. */ + if (ln == NULL) + listAddNodeTail(u->channels,newpat); + else + sdsfree(newpat); + u->flags &= ~USER_FLAG_ALLCHANNELS; } else if (op[0] == '+' && op[1] != '@') { if (strchr(op,'|') == NULL) { if (ACLLookupCommand(op+1) == NULL) { @@ -912,22 +978,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { return C_ERR; } - /* The command should not be set right now in the command - * bitmap, because adding a subcommand of a fully added - * command is probably an error on the user side. */ unsigned long id = ACLGetCommandID(copy); - if (ACLGetUserCommandBit(u,id) == 1) { - zfree(copy); - errno = EBUSY; - return C_ERR; + /* Add the subcommand to the list of valid ones, if the command is not set. */ + if (ACLGetUserCommandBit(u,id) == 0) { + ACLAddAllowedSubcommand(u,id,sub); } - /* Add the subcommand to the list of valid ones. */ - ACLAddAllowedSubcommand(u,id,sub); - - /* We have to clear the command bit so that we force the - * subcommand check. */ - ACLSetUserCommandBit(u,id,0); zfree(copy); } } else if (op[0] == '-' && op[1] != '@') { @@ -947,7 +1003,11 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"reset")) { serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK); serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK); + serverAssert(ACLSetUser(u,"resetchannels",-1) == C_OK); + if (g_pserver->acl_pubsub_default & USER_FLAG_ALLCHANNELS) + serverAssert(ACLSetUser(u,"allchannels",-1) == C_OK); serverAssert(ACLSetUser(u,"off",-1) == C_OK); + serverAssert(ACLSetUser(u,"sanitize-payload",-1) == C_OK); serverAssert(ACLSetUser(u,"-@all",-1) == C_OK); } else { errno = EINVAL; @@ -964,15 +1024,16 @@ const char *ACLSetUserStringError(void) { errmsg = "Unknown command or category name in ACL"; else if (errno == EINVAL) errmsg = "Syntax error"; - else if (errno == EBUSY) - errmsg = "Adding a subcommand of a command already fully " - "added is not allowed. Remove the command to start. " - "Example: -DEBUG +DEBUG|DIGEST"; else if (errno == EEXIST) errmsg = "Adding a pattern after the * pattern (or the " "'allkeys' flag) is not valid and does not have any " "effect. Try 'resetkeys' to start with an empty " "list of patterns"; + else if (errno == EISDIR) + errmsg = "Adding a pattern after the * pattern (or the " + "'allchannels' flag) is not valid and does not have any " + "effect. Try 'resetchannels' to start with an empty " + "list of channels"; else if (errno == ENODEV) errmsg = "The password you are trying to remove from the user does " "not exist"; @@ -988,6 +1049,7 @@ void ACLInitDefaultUser(void) { DefaultUser = ACLCreateUser("default",7); ACLSetUser(DefaultUser,"+@all",-1); ACLSetUser(DefaultUser,"~*",-1); + ACLSetUser(DefaultUser,"&*",-1); ACLSetUser(DefaultUser,"on",-1); ACLSetUser(DefaultUser,"nopass",-1); } @@ -998,7 +1060,6 @@ void ACLInit(void) { UsersToLoad = listCreate(); ACLLog = listCreate(); ACLInitDefaultUser(); - g_pserver->requirepass = NULL; /* Only used for backward compatibility. */ } /* Check the username and password pair and return C_OK if they are valid, @@ -1052,7 +1113,7 @@ int ACLCheckUserCredentials(robj *username, robj *password) { int ACLAuthenticateUser(client *c, robj *username, robj *password) { if (ACLCheckUserCredentials(username,password) == C_OK) { c->authenticated = 1; - c->puser = ACLGetUserByName((sds)ptrFromObj(username),sdslen((sds)ptrFromObj(username))); + c->user = ACLGetUserByName((sds)ptrFromObj(username),sdslen((sds)ptrFromObj(username))); moduleNotifyUserChanged(c); return C_OK; } else { @@ -1068,18 +1129,16 @@ int ACLAuthenticateUser(client *c, robj *username, robj *password) { * command name, so that a command retains the same ID in case of modules that * are unloaded and later reloaded. */ unsigned long ACLGetCommandID(const char *cmdname) { - static rax *map = NULL; - static unsigned long nextid = 0; sds lowername = sdsnew(cmdname); sdstolower(lowername); - if (map == NULL) map = raxNew(); - void *id = raxFind(map,(unsigned char*)lowername,sdslen(lowername)); + if (commandId == NULL) commandId = raxNew(); + void *id = raxFind(commandId,(unsigned char*)lowername,sdslen(lowername)); if (id != raxNotFound) { sdsfree(lowername); return (unsigned long)id; } - raxInsert(map,(unsigned char*)lowername,strlen(lowername), + raxInsert(commandId,(unsigned char*)lowername,strlen(lowername), (void*)nextid,NULL); sdsfree(lowername); unsigned long thisid = nextid; @@ -1097,6 +1156,13 @@ unsigned long ACLGetCommandID(const char *cmdname) { return thisid; } +/* Clear command id table and reset nextid to 0. */ +void ACLClearCommandID(void) { + if (commandId) raxFree(commandId); + commandId = NULL; + nextid = 0; +} + /* Return an username by its name, or NULL if the user does not exist. */ user *ACLGetUserByName(const char *name, size_t namelen) { void *myuser = raxFind(Users,(unsigned char*)name,namelen); @@ -1106,7 +1172,7 @@ user *ACLGetUserByName(const char *name, size_t namelen) { /* Check if the command is ready to be executed in the client 'c', already * referenced by c->cmd, and can be executed by this client according to the - * ACLs associated to the client user c->puser. + * ACLs associated to the client user c->user. * * If the user can execute the command ACL_OK is returned, otherwise * ACL_DENIED_CMD or ACL_DENIED_KEY is returned: the first in case the @@ -1114,15 +1180,15 @@ user *ACLGetUserByName(const char *name, size_t namelen) { * command, the second if the command is denied because the user is trying * to access keys that are not among the specified patterns. */ int ACLCheckCommandPerm(client *c, int *keyidxptr) { - user *u = c->puser; + user *u = c->user; uint64_t id = c->cmd->id; /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; - /* Check if the user can execute this command. */ - if (!(u->flags & USER_FLAG_ALLCOMMANDS) && - c->cmd->proc != authCommand) + /* Check if the user can execute this command or if the command + * doesn't need to be authenticated (hello, auth). */ + if (!(u->flags & USER_FLAG_ALLCOMMANDS) && !(c->cmd->flags & CMD_NO_AUTH)) { /* If the bit is not set we have to check further, in case the * command is allowed just with that specific subcommand. */ @@ -1149,7 +1215,7 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { /* Check if the user can execute commands explicitly touching the keys * mentioned in the command arguments. */ - if (!(c->puser->flags & USER_FLAG_ALLKEYS) && + if (!(c->user->flags & USER_FLAG_ALLKEYS) && (c->cmd->getkeys_proc || c->cmd->firstkey)) { getKeysResult result = GETKEYS_RESULT_INIT; @@ -1187,6 +1253,135 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { return ACL_OK; } +/* Check if the provided channel is whitelisted by the given allowed channels + * list. Glob-style pattern matching is employed, unless the literal flag is + * set. Returns ACL_OK if access is granted or ACL_DENIED_CHANNEL otherwise. */ +int ACLCheckPubsubChannelPerm(sds channel, list *allowed, int literal) { + listIter li; + listNode *ln; + size_t clen = sdslen(channel); + int match = 0; + + listRewind(allowed,&li); + while((ln = listNext(&li))) { + sds pattern = (sds)listNodeValue(ln); + size_t plen = sdslen(pattern); + + if ((literal && !sdscmp(pattern,channel)) || + (!literal && stringmatchlen(pattern,plen,channel,clen,0))) + { + match = 1; + break; + } + } + if (!match) { + return ACL_DENIED_CHANNEL; + } + return ACL_OK; +} + +/* Check if the user's existing pub/sub clients violate the ACL pub/sub + * permissions specified via the upcoming argument, and kill them if so. */ +void ACLKillPubsubClientsIfNeeded(user *u, list *upcoming) { + listIter li, lpi; + listNode *ln, *lpn; + robj *o; + int kill = 0; + + /* Nothing to kill when the upcoming are a literal super set of the original + * permissions. */ + listRewind(u->channels,&li); + while (!kill && ((ln = listNext(&li)) != NULL)) { + sds pattern = (sds)listNodeValue(ln); + kill = (ACLCheckPubsubChannelPerm(pattern,upcoming,1) == + ACL_DENIED_CHANNEL); + } + if (!kill) return; + + /* Scan all connected clients to find the user's pub/subs. */ + listRewind(g_pserver->clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = (client*)listNodeValue(ln); + kill = 0; + + if (c->user == u && getClientType(c) == CLIENT_TYPE_PUBSUB) { + /* Check for pattern violations. */ + listRewind(c->pubsub_patterns,&lpi); + while (!kill && ((lpn = listNext(&lpi)) != NULL)) { + o = (robj*)lpn->value; + kill = (ACLCheckPubsubChannelPerm(szFromObj(o),upcoming,1) == + ACL_DENIED_CHANNEL); + } + /* Check for channel violations. */ + if (!kill) { + dictIterator *di = dictGetIterator(c->pubsub_channels); + dictEntry *de; + while (!kill && ((de = dictNext(di)) != NULL)) { + o = (robj*)dictGetKey(de); + kill = (ACLCheckPubsubChannelPerm(szFromObj(o),upcoming,0) == + ACL_DENIED_CHANNEL); + } + dictReleaseIterator(di); + } + + /* Kill it. */ + if (kill) { + freeClientAsync(c); + } + } + } +} + +/* Check if the pub/sub channels of the command, that's ready to be executed in + * the client 'c', can be executed by this client according to the ACLs channels + * associated to the client user c->user. + * + * idx and count are the index and count of channel arguments from the + * command. The literal argument controls whether the user's ACL channels are + * evaluated as literal values or matched as glob-like patterns. + * + * If the user can execute the command ACL_OK is returned, otherwise + * ACL_DENIED_CHANNEL. */ +int ACLCheckPubsubPerm(client *c, int idx, int count, int literal, int *idxptr) { + user *u = c->user; + + /* If there is no associated user, the connection can run anything. */ + if (u == NULL) return ACL_OK; + + /* Check if the user can access the channels mentioned in the command's + * arguments. */ + if (!(c->user->flags & USER_FLAG_ALLCHANNELS)) { + for (int j = idx; j < idx+count; j++) { + if (ACLCheckPubsubChannelPerm(szFromObj(c->argv[j]),u->channels,literal) + != ACL_OK) { + if (idxptr) *idxptr = j; + return ACL_DENIED_CHANNEL; + } + } + } + + /* If we survived all the above checks, the user can execute the + * command. */ + return ACL_OK; + +} + +/* Check whether the command is ready to be exceuted by ACLCheckCommandPerm. + * If check passes, then check whether pub/sub channels of the command is + * ready to be executed by ACLCheckPubsubPerm */ +int ACLCheckAllPerm(client *c, int *idxptr) { + int acl_retval = ACLCheckCommandPerm(c,idxptr); + if (acl_retval != ACL_OK) + return acl_retval; + if (c->cmd->proc == publishCommand) + acl_retval = ACLCheckPubsubPerm(c,1,1,0,idxptr); + else if (c->cmd->proc == subscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,0,idxptr); + else if (c->cmd->proc == psubscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,1,idxptr); + return acl_retval; +} + /* ============================================================================= * ACL loading / saving functions * ==========================================================================*/ @@ -1546,7 +1741,7 @@ void ACLLoadUsersAtStartup(void) { sds errors = ACLLoadFromFile(g_pserver->acl_filename); if (errors) { serverLog(LL_WARNING, - "Aborting Redis startup because of ACL errors: %s", errors); + "Aborting KeyDB startup because of ACL errors: %s", errors); sdsfree(errors); exit(1); } @@ -1588,8 +1783,8 @@ int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { } /* Release an ACL log entry. */ -void ACLFreeLogEntry(ACLLogEntry *leptr) { - ACLLogEntry *le = leptr; +void ACLFreeLogEntry(const void *leptr) { + ACLLogEntry *le = (ACLLogEntry*)leptr; sdsfree(le->object); sdsfree(le->username); sdsfree(le->cinfo); @@ -1602,24 +1797,25 @@ void ACLFreeLogEntry(ACLLogEntry *leptr) { * the log entry instead of creating many entries for very similar ACL * rules issues. * - * The keypos argument is only used when the reason is ACL_DENIED_KEY, since - * it allows the function to log the key name that caused the problem. - * Similarly the username is only passed when we failed to authenticate the - * user with AUTH or HELLO, for the ACL_DENIED_AUTH reason. Otherwise - * it will just be NULL. + * The argpos argument is used when the reason is ACL_DENIED_KEY or + * ACL_DENIED_CHANNEL, since it allows the function to log the key or channel + * name that caused the problem. Similarly the username is only passed when we + * failed to authenticate the user with AUTH or HELLO, for the ACL_DENIED_AUTH + * reason. Otherwise it will just be NULL. */ -void addACLLogEntry(client *c, int reason, int keypos, sds username) { +void addACLLogEntry(client *c, int reason, int argpos, sds username) { /* Create a new entry. */ struct ACLLogEntry *le = (ACLLogEntry*)zmalloc(sizeof(*le)); le->count = 1; le->reason = reason; - le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->puser->name); + le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->user->name); le->ctime = mstime(); switch(reason) { case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break; - case ACL_DENIED_KEY: le->object = sdsnew(szFromObj(c->argv[keypos])); break; - case ACL_DENIED_AUTH: le->object = sdsnew(szFromObj(c->argv[0])); break; + case ACL_DENIED_KEY: le->object = sdsdup(szFromObj(c->argv[argpos])); break; + case ACL_DENIED_CHANNEL: le->object = sdsdup(szFromObj(c->argv[argpos])); break; + case ACL_DENIED_AUTH: le->object = sdsdup(szFromObj(c->argv[0])); break; default: le->object = sdsempty(); } @@ -1666,7 +1862,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) { le->cinfo = NULL; ACLFreeLogEntry(le); } else { - /* Add it to our list of entires. We'll have to trim the list + /* Add it to our list of entries. We'll have to trim the list * to its maximum size. */ listAddNodeHead(ACLLog, le); while(listLength(ACLLog) > g_pserver->acllog_max_len) { @@ -1699,6 +1895,10 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) { void aclCommand(client *c) { char *sub = szFromObj(c->argv[1]); if (!strcasecmp(sub,"setuser") && c->argc >= 3) { + /* Consider information about passwords or permissions + * to be sensitive, which will be the arguments for this + * subcommand. */ + preventCommandLogging(c); sds username = szFromObj(c->argv[2]); /* Check username validity. */ if (ACLStringHasSpaces(username,sdslen(username))) { @@ -1727,6 +1927,11 @@ void aclCommand(client *c) { } } + /* Existing pub/sub clients authenticated with the user may need to be + * disconnected if (some of) their channel permissions were revoked. */ + if (u && !(tempu->flags & USER_FLAG_ALLCHANNELS)) + ACLKillPubsubClientsIfNeeded(u,tempu->channels); + /* Overwrite the user with the temporary user we modified above. */ if (!u) u = ACLCreateUser(username,sdslen(username)); serverAssert(u != NULL); @@ -1762,7 +1967,7 @@ void aclCommand(client *c) { return; } - addReplyMapLen(c,4); + addReplyMapLen(c,5); /* Flags */ addReplyBulkCString(c,"flags"); @@ -1807,6 +2012,22 @@ void aclCommand(client *c) { addReplyBulkCBuffer(c,thispat,sdslen(thispat)); } } + + /* Pub/sub patterns */ + addReplyBulkCString(c,"channels"); + if (u->flags & USER_FLAG_ALLCHANNELS) { + addReplyArrayLen(c,1); + addReplyBulkCBuffer(c,"*",1); + } else { + addReplyArrayLen(c,listLength(u->channels)); + listIter li; + listNode *ln; + listRewind(u->channels,&li); + while((ln = listNext(&li))) { + sds thispat = (sds)listNodeValue(ln); + addReplyBulkCBuffer(c,thispat,sdslen(thispat)); + } + } } else if ((!strcasecmp(sub,"list") || !strcasecmp(sub,"users")) && c->argc == 2) { @@ -1832,15 +2053,15 @@ void aclCommand(client *c) { } raxStop(&ri); } else if (!strcasecmp(sub,"whoami") && c->argc == 2) { - if (c->puser != NULL) { - addReplyBulkCBuffer(c,c->puser->name,sdslen(c->puser->name)); + if (c->user != NULL) { + addReplyBulkCBuffer(c,c->user->name,sdslen(c->user->name)); } else { addReplyNull(c); } } else if (g_pserver->acl_filename[0] == '\0' && (!strcasecmp(sub,"load") || !strcasecmp(sub,"save"))) { - addReplyError(c,"This Redis instance is not configured to use an ACL file. You may want to specify users via the ACL SETUSER command and then issue a CONFIG REWRITE (assuming you have a Redis configuration file set) in order to store users in the Redis configuration."); + addReplyError(c,"This KeyDB instance is not configured to use an ACL file. You may want to specify users via the ACL SETUSER command and then issue a CONFIG REWRITE (assuming you have a KeyDB configuration file set) in order to store users in the KeyDB configuration."); return; } else if (!strcasecmp(sub,"load") && c->argc == 2) { sds errors = ACLLoadFromFile(g_pserver->acl_filename); @@ -1911,7 +2132,7 @@ void aclCommand(client *c) { * the "RESET" command in order to flush the old entries. */ if (c->argc == 3) { if (!strcasecmp(szFromObj(c->argv[2]),"reset")) { - listSetFreeMethod(ACLLog,(void(*)(const void*))ACLFreeLogEntry); + listSetFreeMethod(ACLLog,ACLFreeLogEntry); listEmpty(ACLLog); listSetFreeMethod(ACLLog,NULL); addReply(c,shared.ok); @@ -1940,10 +2161,11 @@ void aclCommand(client *c) { addReplyLongLong(c,le->count); addReplyBulkCString(c,"reason"); - const char *reasonstr = "INVALID_REASON"; + const char *reasonstr; switch(le->reason) { case ACL_DENIED_CMD: reasonstr="command"; break; case ACL_DENIED_KEY: reasonstr="key"; break; + case ACL_DENIED_CHANNEL: reasonstr="channel"; break; case ACL_DENIED_AUTH: reasonstr="auth"; break; default: reasonstr="unknown"; } @@ -1971,18 +2193,30 @@ void aclCommand(client *c) { } } else if (c->argc == 2 && !strcasecmp(sub,"help")) { const char *help[] = { -"LOAD -- Reload users from the ACL file.", -"SAVE -- Save the current config to the ACL file.", -"LIST -- Show user details in config file format.", -"USERS -- List all the registered usernames.", -"SETUSER [attribs ...] -- Create or modify a user.", -"GETUSER -- Get the user details.", -"DELUSER [...] -- Delete a list of users.", -"CAT -- List available categories.", -"CAT -- List commands inside category.", -"GENPASS [] -- Generate a secure user password.", -"WHOAMI -- Return the current connection username.", -"LOG [ | RESET] -- Show the ACL log entries.", +"CAT []", +" List all commands that belong to , or all command categories", +" when no category is specified.", +"DELUSER [ ...]", +" Delete a list of users.", +"GETUSER ", +" Get the user's details.", +"GENPASS []", +" Generate a secure 256-bit user password. The optional `bits` argument can", +" be used to specify a different size.", +"LIST", +" Show users details in config file format.", +"LOAD", +" Reload users from the ACL file.", +"LOG [ | RESET]", +" Show the ACL log entries.", +"SAVE", +" Save the current config to the ACL file.", +"SETUSER [ ...]", +" Create or modify a user with the specified attributes.", +"USERS", +" List all the registered usernames.", +"WHOAMI", +" Return the current connection username.", NULL }; addReplyHelp(c,help); @@ -2011,7 +2245,7 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd) { void authCommand(client *c) { /* Only two or three argument forms are allowed. */ if (c->argc > 3) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -2028,7 +2262,7 @@ void authCommand(client *c) { return; } - username = createStringObject("default",7); + username = shared.default_username; password = c->argv[1]; } else { username = c->argv[1]; @@ -2038,11 +2272,19 @@ void authCommand(client *c) { if (ACLAuthenticateUser(c,username,password) == C_OK) { addReply(c,shared.ok); } else { - addReplyError(c,"-WRONGPASS invalid username-password pair"); + addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled."); } - - /* Free the "default" string object we created for the two - * arguments form. */ - if (c->argc == 2) decrRefCount(username); } +/* Set the password for the "default" ACL user. This implements supports for + * requirepass config, so passing in NULL will set the user to be nopass. */ +void ACLUpdateDefaultUserPassword(sds password) { + ACLSetUser(DefaultUser,"resetpass",-1); + if (password) { + sds aclop = sdscatlen(sdsnew(">"), password, sdslen(password)); + ACLSetUser(DefaultUser,aclop,sdslen(aclop)); + sdsfree(aclop); + } else { + ACLSetUser(DefaultUser,"nopass",-1); + } +} diff --git a/src/adlist.c b/src/adlist.c index 6d5d77fb3..1adb0bcb0 100644 --- a/src/adlist.c +++ b/src/adlist.c @@ -360,15 +360,16 @@ void listRotateHeadToTail(list *list) { /* Add all the elements of the list 'o' at the end of the * list 'l'. The list 'other' remains empty but otherwise valid. */ void listJoin(list *l, list *o) { - if (o->head) - o->head->prev = l->tail; + if (o->len == 0) return; + + o->head->prev = l->tail; if (l->tail) l->tail->next = o->head; else l->head = o->head; - if (o->tail) l->tail = o->tail; + l->tail = o->tail; l->len += o->len; /* Setup other as an empty list. */ diff --git a/src/ae.cpp b/src/ae.cpp index f96ef4f6c..d96eee7bc 100644 --- a/src/ae.cpp +++ b/src/ae.cpp @@ -30,6 +30,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "ae.h" +#include "anet.h" +#include "fastlock.h" + #include #include #include @@ -274,12 +278,13 @@ aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; int i; + monotonicInit(); /* just in case the calling app didn't initialize */ + if ((eventLoop = (aeEventLoop*)zmalloc(sizeof(*eventLoop), MALLOC_LOCAL)) == NULL) goto err; eventLoop->events = (aeFileEvent*)zmalloc(sizeof(aeFileEvent)*setsize, MALLOC_LOCAL); eventLoop->fired = (aeFiredEvent*)zmalloc(sizeof(aeFiredEvent)*setsize, MALLOC_LOCAL); if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; eventLoop->setsize = setsize; - eventLoop->lastTime = time(NULL); eventLoop->timeEventHead = NULL; eventLoop->timeEventNextId = 0; eventLoop->stop = 0; @@ -451,29 +456,6 @@ extern "C" int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { return fe->mask; } -static void aeGetTime(long *seconds, long *milliseconds) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - *seconds = tv.tv_sec; - *milliseconds = tv.tv_usec/1000; -} - -static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) { - long cur_sec, cur_ms, when_sec, when_ms; - - aeGetTime(&cur_sec, &cur_ms); - when_sec = cur_sec + milliseconds/1000; - when_ms = cur_ms + milliseconds%1000; - if (when_ms >= 1000) { - when_sec ++; - when_ms -= 1000; - } - *sec = when_sec; - *ms = when_ms; -} - extern "C" long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, aeTimeProc *proc, void *clientData, aeEventFinalizerProc *finalizerProc) @@ -485,7 +467,7 @@ extern "C" long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long millise te = (aeTimeEvent*)zmalloc(sizeof(*te), MALLOC_LOCAL); if (te == NULL) return AE_ERR; te->id = id; - aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms); + te->when = getMonotonicUs() + milliseconds * 1000; te->timeProc = proc; te->finalizerProc = finalizerProc; te->clientData = clientData; @@ -512,10 +494,8 @@ extern "C" int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) return AE_ERR; /* NO event with the specified ID found */ } -/* Search the first timer to fire. - * This operation is useful to know how many time the select can be - * put in sleep without to delay any event. - * If there are no timers NULL is returned. +/* How many microseconds until the first timer should fire. + * If there are no timers, -1 is returned. * * Note that's O(N) since time events are unsorted. * Possible optimizations (not needed by Redis so far, but...): @@ -523,20 +503,19 @@ extern "C" int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) * Much better but still insertion or deletion of timers is O(N). * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). */ -static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop) -{ - serverAssert(g_eventLoopThisThread == NULL || g_eventLoopThisThread == eventLoop); +static int64_t usUntilEarliestTimer(aeEventLoop *eventLoop) { aeTimeEvent *te = eventLoop->timeEventHead; - aeTimeEvent *nearest = NULL; + if (te == NULL) return -1; - while(te) { - if (!nearest || te->when_sec < nearest->when_sec || - (te->when_sec == nearest->when_sec && - te->when_ms < nearest->when_ms)) - nearest = te; + aeTimeEvent *earliest = NULL; + while (te) { + if (!earliest || te->when < earliest->when) + earliest = te; te = te->next; } - return nearest; + + monotime now = getMonotonicUs(); + return (now >= earliest->when) ? 0 : earliest->when - now; } /* Process time events */ @@ -545,29 +524,11 @@ static int processTimeEvents(aeEventLoop *eventLoop) { int processed = 0; aeTimeEvent *te; long long maxId; - time_t now = time(NULL); - - /* If the system clock is moved to the future, and then set back to the - * right value, time events may be delayed in a random way. Often this - * means that scheduled operations will not be performed soon enough. - * - * Here we try to detect system clock skews, and force all the time - * events to be processed ASAP when this happens: the idea is that - * processing events earlier is less dangerous than delaying them - * indefinitely, and practice suggests it is. */ - if (now < eventLoop->lastTime) { - te = eventLoop->timeEventHead; - while(te) { - te->when_sec = 0; - te = te->next; - } - } - eventLoop->lastTime = now; te = eventLoop->timeEventHead; maxId = eventLoop->timeEventNextId-1; + monotime now = getMonotonicUs(); while(te) { - long now_sec, now_ms; long long id; /* Remove events scheduled for deletion. */ @@ -589,6 +550,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) { if (te->finalizerProc) { if (!ulock.owns_lock()) ulock.lock(); te->finalizerProc(eventLoop, te->clientData); + now = getMonotonicUs(); } zfree(te); te = next; @@ -604,10 +566,8 @@ static int processTimeEvents(aeEventLoop *eventLoop) { te = te->next; continue; } - aeGetTime(&now_sec, &now_ms); - if (now_sec > te->when_sec || - (now_sec == te->when_sec && now_ms >= te->when_ms)) - { + + if (te->when <= now) { if (!ulock.owns_lock()) ulock.lock(); int retval; @@ -616,8 +576,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) { retval = te->timeProc(eventLoop, id, te->clientData); te->refcount--; processed++; + now = getMonotonicUs(); if (retval != AE_NOMORE) { - aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms); + te->when = now + retval * 1000; } else { te->id = AE_DELETED_EVENT_ID; } @@ -707,37 +668,23 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* Nothing to do? return ASAP */ if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0; - /* Note that we want call select() even if there are no + /* Note that we want to call select() even if there are no * file events to process as long as we want to process time * events, in order to sleep until the next time event is ready * to fire. */ if (eventLoop->maxfd != -1 || ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { int j; - aeTimeEvent *shortest = NULL; struct timeval tv, *tvp; + int64_t usUntilTimer = -1; if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) - shortest = aeSearchNearestTimer(eventLoop); - if (shortest) { - long now_sec, now_ms; + usUntilTimer = usUntilEarliestTimer(eventLoop); - aeGetTime(&now_sec, &now_ms); + if (usUntilTimer >= 0) { + tv.tv_sec = usUntilTimer / 1000000; + tv.tv_usec = usUntilTimer % 1000000; tvp = &tv; - - /* How many milliseconds we need to wait for the next - * time event to fire? */ - long long ms = - (shortest->when_sec - now_sec)*1000 + - shortest->when_ms - now_ms; - - if (ms > 0) { - tvp->tv_sec = ms/1000; - tvp->tv_usec = (ms % 1000)*1000; - } else { - tvp->tv_sec = 0; - tvp->tv_usec = 0; - } } else { /* If we have to check for events but need to return * ASAP because of AE_DONT_WAIT we need to set the timeout @@ -819,14 +766,8 @@ void aeMain(aeEventLoop *eventLoop) { eventLoop->stop = 0; g_eventLoopThisThread = eventLoop; while (!eventLoop->stop) { - if (eventLoop->beforesleep != NULL) { - std::unique_lock ulock(g_lock, std::defer_lock); - if (!(eventLoop->beforesleepFlags & AE_SLEEP_THREADSAFE)) - ulock.lock(); - eventLoop->beforesleep(eventLoop); - } serverAssert(!aeThreadOwnsLock()); // we should have relinquished it after processing - aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP); + aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP); serverAssert(!aeThreadOwnsLock()); // we should have relinquished it after processing } } diff --git a/src/ae.h b/src/ae.h index aec1df154..477e274f1 100644 --- a/src/ae.h +++ b/src/ae.h @@ -36,7 +36,7 @@ #ifdef __cplusplus #include #endif -#include +#include "monotonic.h" #include "fastlock.h" #ifdef __cplusplus @@ -91,8 +91,7 @@ typedef struct aeFileEvent { /* Time event structure */ typedef struct aeTimeEvent { long long id; /* time event identifier. */ - long when_sec; /* seconds */ - long when_ms; /* milliseconds */ + monotime when; aeTimeProc *timeProc; aeEventFinalizerProc *finalizerProc; void *clientData; @@ -113,7 +112,6 @@ typedef struct aeEventLoop { int maxfd; /* highest file descriptor currently registered */ int setsize; /* max number of file descriptors tracked */ long long timeEventNextId; - time_t lastTime; /* Used to detect system clock skew */ aeFileEvent *events; /* Registered events */ aeFiredEvent *fired; /* Fired events */ aeTimeEvent *timeEventHead; diff --git a/src/ae_epoll.cpp b/src/ae_epoll.cpp index 452fc49ae..76cfa680a 100644 --- a/src/ae_epoll.cpp +++ b/src/ae_epoll.cpp @@ -51,6 +51,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->epfd); eventLoop->apidata = state; return 0; } @@ -114,7 +115,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { int retval, numevents = 0; retval = epoll_wait(state->epfd,state->events,eventLoop->setsize, - tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); + tvp ? (tvp->tv_sec*1000 + (tvp->tv_usec + 999)/1000) : -1); if (retval > 0) { int j; diff --git a/src/ae_evport.c b/src/ae_evport.c index 744e1a6bb..de02093ac 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -67,7 +67,7 @@ static int evport_debug = 0; typedef struct aeApiState { int portfd; /* event port */ - int npending; /* # of pending fds */ + uint_t npending; /* # of pending fds */ int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ } aeApiState; @@ -82,6 +82,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->portfd); state->npending = 0; @@ -95,6 +96,8 @@ static int aeApiCreate(aeEventLoop *eventLoop) { } static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + (void) eventLoop; + (void) setsize; /* Nothing to resize here. */ return 0; } @@ -107,7 +110,7 @@ static void aeApiFree(aeEventLoop *eventLoop) { } static int aeApiLookupPending(aeApiState *state, int fd) { - int i; + uint_t i; for (i = 0; i < state->npending; i++) { if (state->pending_fds[i] == fd) @@ -243,7 +246,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { aeApiState *state = eventLoop->apidata; struct timespec timeout, *tsp; - int mask, i; + uint_t mask, i; uint_t nevents; port_event_t event[MAX_EVENT_BATCHSZ]; diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 5c83f6464..c6e9cf3fa 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -53,6 +53,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->kqfd); eventLoop->apidata = state; return 0; } diff --git a/src/anet.c b/src/anet.c index 8b8b6ae33..0b3e462c3 100644 --- a/src/anet.c +++ b/src/anet.c @@ -69,6 +69,11 @@ int anetSetBlock(char *err, int fd, int non_block) { return ANET_ERR; } + /* Check if this flag has been set or unset, if so, + * then there is no need to call fcntl to set/unset it again. */ + if (!!(flags & O_NONBLOCK) == !!non_block) + return ANET_OK; + if (non_block) flags |= O_NONBLOCK; else @@ -89,6 +94,29 @@ int anetBlock(char *err, int fd) { return anetSetBlock(err,fd,0); } +/* Enable the FD_CLOEXEC on the given fd to avoid fd leaks. + * This function should be invoked for fd's on specific places + * where fork + execve system calls are called. */ +int anetCloexec(int fd) { + int r; + int flags; + + do { + r = fcntl(fd, F_GETFD); + } while (r == -1 && errno == EINTR); + + if (r == -1 || (r & FD_CLOEXEC)) + return r; + + flags = r | FD_CLOEXEC; + + do { + r = fcntl(fd, F_SETFD, flags); + } while (r == -1 && errno == EINTR); + + return r; +} + /* Set TCP keep alive option to detect dead peers. The interval option * is only used for Linux as we are using Linux-specific APIs to set * the probe send time, interval, and count. */ @@ -158,27 +186,6 @@ int anetDisableTcpNoDelay(char *err, int fd) return anetSetTcpNoDelay(err, fd, 0); } - -int anetSetSendBuffer(char *err, int fd, int buffsize) -{ - if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffsize, sizeof(buffsize)) == -1) - { - anetSetError(err, "setsockopt SO_SNDBUF: %s", strerror(errno)); - return ANET_ERR; - } - return ANET_OK; -} - -int anetTcpKeepAlive(char *err, int fd) -{ - int yes = 1; - if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)) == -1) { - anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno)); - return ANET_ERR; - } - return ANET_OK; -} - /* Set the socket send timeout (SO_SNDTIMEO socket option) to the specified * number of milliseconds, or disable it if the 'ms' argument is zero. */ int anetSendTimeout(char *err, int fd, long long ms) { @@ -207,14 +214,13 @@ int anetRecvTimeout(char *err, int fd, long long ms) { return ANET_OK; } -/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to - * do the actual work. It resolves the hostname "host" and set the string - * representation of the IP address into the buffer pointed by "ipbuf". +/* Resolve the hostname "host" and set the string representation of the + * IP address into the buffer pointed by "ipbuf". * * If flags is set to ANET_IP_ONLY the function only resolves hostnames * that are actually already IPv4 or IPv6 addresses. This turns the function * into a validating / normalizing function. */ -int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags) { struct addrinfo hints, *info; @@ -241,14 +247,6 @@ int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, return ANET_OK; } -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_NONE); -} - -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_IP_ONLY); -} - static int anetSetReuseAddr(char *err, int fd) { int yes = 1; /* Make sure connection-intensive things like the redis benchmark @@ -373,23 +371,11 @@ end: } } -int anetTcpConnect(char *err, const char *addr, int port) -{ - return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); -} - int anetTcpNonBlockConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, - const char *source_addr) -{ - return anetTcpGenericConnect(err,addr,port,source_addr, - ANET_CONNECT_NONBLOCK); -} - int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr) { @@ -425,46 +411,6 @@ int anetUnixGenericConnect(char *err, const char *path, int flags) return s; } -int anetUnixConnect(char *err, const char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); -} - -int anetUnixNonBlockConnect(char *err, const char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); -} - -/* Like read(2) but make sure 'count' is read before to return - * (unless error or EOF condition is encountered) */ -int anetRead(int fd, char *buf, int count) -{ - ssize_t nread, totlen = 0; - while(totlen != count) { - nread = read(fd,buf,count-totlen); - if (nread == 0) return totlen; - if (nread == -1) return -1; - totlen += nread; - buf += nread; - } - return totlen; -} - -/* Like write(2) but make sure 'count' is written before to return - * (unless error is encountered) */ -int anetWrite(int fd, char *buf, int count) -{ - ssize_t nwritten, totlen = 0; - while(totlen != count) { - nwritten = write(fd,buf,count-totlen); - if (nwritten == 0) return totlen; - if (nwritten == -1) return -1; - totlen += nwritten; - buf += nwritten; - } - return totlen; -} - static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int backlog) { if (bind(s,sa,len) == -1) { anetSetError(err, "bind: %s", strerror(errno)); @@ -484,13 +430,12 @@ static int anetV6Only(char *err, int s) { int yes = 1; if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) { anetSetError(err, "setsockopt: %s", strerror(errno)); - close(s); return ANET_ERR; } return ANET_OK; } -static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backlog, int fReusePort, int fFirstListen) +static int _anetTcpServer(char *err, int port, const char *bindaddr, int af, int backlog, int fReusePort, int fFirstListen) { int s = -1, rv; char _port[6]; /* strlen("65535") */ @@ -501,6 +446,10 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backl hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; /* No effect if bindaddr != NULL */ + if (bindaddr && !strcmp("*", bindaddr)) + bindaddr = NULL; + if (af == AF_INET6 && bindaddr && !strcmp("::*", bindaddr)) + bindaddr = NULL; if ((rv = getaddrinfo(bindaddr,_port,&hints,&servinfo)) != 0) { anetSetError(err, "%s", gai_strerror(rv)); @@ -533,12 +482,12 @@ end: return s; } -int anetTcpServer(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen) +int anetTcpServer(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen) { return _anetTcpServer(err, port, bindaddr, AF_INET, backlog, fReusePort, fFirstListen); } -int anetTcp6Server(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen) +int anetTcp6Server(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen) { return _anetTcpServer(err, port, bindaddr, AF_INET6, backlog, fReusePort, fFirstListen); } @@ -607,11 +556,15 @@ int anetUnixAccept(char *err, int s) { return fd; } -int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type) { struct sockaddr_storage sa; socklen_t salen = sizeof(sa); - if (getpeername(fd,(struct sockaddr*)&sa,&salen) == -1) goto error; + if (fd_to_str_type == FD_TO_PEER_NAME) { + if (getpeername(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } else { + if (getsockname(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } if (ip_len == 0) goto error; if (sa.ss_family == AF_INET) { @@ -623,7 +576,7 @@ int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); if (port) *port = ntohs(s->sin6_port); } else if (sa.ss_family == AF_UNIX) { - if (ip) strncpy(ip,"/unixsocket",ip_len); + if (ip) snprintf(ip, ip_len, "/unixsocket"); if (port) *port = 0; } else { goto error; @@ -651,41 +604,11 @@ int anetFormatAddr(char *buf, size_t buf_len, char *ip, int port) { "[%s]:%d" : "%s:%d", ip, port); } -/* Like anetFormatAddr() but extract ip and port from the socket's peer. */ -int anetFormatPeer(int fd, char *buf, size_t buf_len) { +/* Like anetFormatAddr() but extract ip and port from the socket's peer/sockname. */ +int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type) { char ip[INET6_ADDRSTRLEN]; int port; - anetPeerToString(fd,ip,sizeof(ip),&port); + anetFdToString(fd,ip,sizeof(ip),&port,fd_to_str_type); return anetFormatAddr(buf, buf_len, ip, port); } - -int anetSockName(int fd, char *ip, size_t ip_len, int *port) { - struct sockaddr_storage sa; - socklen_t salen = sizeof(sa); - - if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) { - if (port) *port = 0; - ip[0] = '?'; - ip[1] = '\0'; - return -1; - } - if (sa.ss_family == AF_INET) { - struct sockaddr_in *s = (struct sockaddr_in *)&sa; - if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); - if (port) *port = ntohs(s->sin_port); - } else { - struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; - if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); - if (port) *port = ntohs(s->sin6_port); - } - return 0; -} - -int anetFormatSock(int fd, char *fmt, size_t fmt_len) { - char ip[INET6_ADDRSTRLEN]; - int port; - - anetSockName(fd,ip,sizeof(ip),&port); - return anetFormatAddr(fmt, fmt_len, ip, port); -} diff --git a/src/anet.h b/src/anet.h index b31a38ad6..0f6cf31ba 100644 --- a/src/anet.h +++ b/src/anet.h @@ -53,34 +53,29 @@ extern "C" { #undef ip_len #endif -int anetTcpConnect(char *err, const char *addr, int port); +/* FD to address string conversion types */ +#define FD_TO_PEER_NAME 0 +#define FD_TO_SOCK_NAME 1 + int anetTcpNonBlockConnect(char *err, const char *addr, int port); -int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr); int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); -int anetUnixConnect(char *err, const char *path); -int anetUnixNonBlockConnect(char *err, const char *path); -int anetRead(int fd, char *buf, int count); -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); -int anetTcpServer(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen); -int anetTcp6Server(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen); +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); +int anetTcpServer(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen); +int anetTcp6Server(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen); int anetUnixServer(char *err, char *path, mode_t perm, int backlog); int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); -int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetBlock(char *err, int fd); +int anetCloexec(int fd); int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); -int anetTcpKeepAlive(char *err, int fd); int anetSendTimeout(char *err, int fd, long long ms); int anetRecvTimeout(char *err, int fd, long long ms); -int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type); int anetKeepAlive(char *err, int fd, int interval); -int anetSockName(int fd, char *ip, size_t ip_len, int *port); int anetFormatAddr(char *fmt, size_t fmt_len, char *ip, int port); -int anetFormatPeer(int fd, char *fmt, size_t fmt_len); -int anetFormatSock(int fd, char *fmt, size_t fmt_len); +int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type); #ifdef __cplusplus } diff --git a/src/aof.cpp b/src/aof.cpp index e8f77e879..326c30ba2 100644 --- a/src/aof.cpp +++ b/src/aof.cpp @@ -221,29 +221,27 @@ int aofFsyncInProgress(void) { /* Starts a background task that performs fsync() against the specified * file descriptor (the one of the AOF file) in another thread. */ void aof_background_fsync(int fd) { - bioCreateBackgroundJob(BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL); + bioCreateFsyncJob(fd); } /* Kills an AOFRW child process if exists */ void killAppendOnlyChild(void) { int statloc; /* No AOFRW child? return. */ - if (g_pserver->aof_child_pid == -1) return; + if (g_pserver->child_type != CHILD_TYPE_AOF) return; /* Kill AOFRW child, wait for child exit. */ serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld", - (long) g_pserver->aof_child_pid); - if (kill(g_pserver->aof_child_pid,SIGUSR1) != -1) { - while(wait3(&statloc,0,NULL) != g_pserver->aof_child_pid); + (long) g_pserver->child_pid); + if (kill(g_pserver->child_pid,SIGUSR1) != -1) { + while(waitpid(-1, &statloc, 0) != g_pserver->child_pid); } /* Reset the buffer accumulating changes while the child saves. */ aofRewriteBufferReset(); - aofRemoveTempFile(g_pserver->aof_child_pid); - g_pserver->aof_child_pid = -1; + aofRemoveTempFile(g_pserver->child_pid); + resetChildState(); g_pserver->aof_rewrite_time_start = -1; /* Close pipes used for IPC between the two processes. */ aofClosePipes(); - closeChildInfoPipe(); - updateDictResizePolicy(); } /* Called when the user switches from "appendonly yes" to "appendonly no" @@ -251,7 +249,12 @@ void killAppendOnlyChild(void) { void stopAppendOnly(void) { serverAssert(g_pserver->aof_state != AOF_OFF); flushAppendOnlyFile(1); - redis_fsync(g_pserver->aof_fd); + if (redis_fsync(g_pserver->aof_fd) == -1) { + serverLog(LL_WARNING,"Fail to fsync the AOF file: %s",strerror(errno)); + } else { + g_pserver->aof_fsync_offset = g_pserver->aof_current_size; + g_pserver->aof_last_fsync = g_pserver->unixtime; + } close(g_pserver->aof_fd); g_pserver->aof_fd = -1; @@ -259,6 +262,8 @@ void stopAppendOnly(void) { g_pserver->aof_state = AOF_OFF; g_pserver->aof_rewrite_scheduled = 0; killAppendOnlyChild(); + sdsfree(g_pserver->aof_buf); + g_pserver->aof_buf = sdsempty(); } /* Called when the user switches from "appendonly no" to "appendonly yes" @@ -273,27 +278,27 @@ int startAppendOnly(void) { char *cwdp = getcwd(cwd,MAXPATHLEN); serverLog(LL_WARNING, - "Redis needs to enable the AOF but can't open the " + "KeyDB needs to enable the AOF but can't open the " "append only file %s (in server root dir %s): %s", g_pserver->aof_filename, cwdp ? cwdp : "unknown", strerror(errno)); return C_ERR; } - if (hasActiveChildProcess() && g_pserver->aof_child_pid == -1) { + if (hasActiveChildProcess() && g_pserver->child_type != CHILD_TYPE_AOF) { g_pserver->aof_rewrite_scheduled = 1; serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not * accumulating the AOF buffer. */ - if (g_pserver->aof_child_pid != -1) { + if (g_pserver->child_type == CHILD_TYPE_AOF) { serverLog(LL_WARNING,"AOF was enabled but there is already an AOF rewriting in background. Stopping background AOF and starting a rewrite now."); killAppendOnlyChild(); } if (rewriteAppendOnlyFileBackground() == C_ERR) { close(newfd); - serverLog(LL_WARNING,"Redis needs to enable the AOF but can't trigger a background AOF rewrite operation. Check the above logs for more info about the error."); + serverLog(LL_WARNING,"KeyDB needs to enable the AOF but can't trigger a background AOF rewrite operation. Check the above logs for more info about the error."); return C_ERR; } } @@ -302,6 +307,21 @@ int startAppendOnly(void) { g_pserver->aof_state = AOF_WAIT_REWRITE; g_pserver->aof_last_fsync = g_pserver->unixtime; g_pserver->aof_fd = newfd; + + /* If AOF fsync error in bio job, we just ignore it and log the event. */ + int aof_bio_fsync_status; + atomicGet(g_pserver->aof_bio_fsync_status, aof_bio_fsync_status); + if (aof_bio_fsync_status == C_ERR) { + serverLog(LL_WARNING, + "AOF reopen, just ignore the AOF fsync error in bio job"); + atomicSet(g_pserver->aof_bio_fsync_status,C_OK); + } + + /* If AOF was in error state, we just ignore it and log the event. */ + if (g_pserver->aof_last_write_status == C_ERR) { + serverLog(LL_WARNING,"AOF reopen, just ignore the last error."); + g_pserver->aof_last_write_status = C_OK; + } return C_OK; } @@ -395,7 +415,7 @@ void flushAppendOnlyFile(int force) { /* Otherwise fall trough, and go write since we can't wait * over two seconds. */ g_pserver->aof_delayed_fsync++; - serverLog(LL_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down Redis."); + serverLog(LL_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down KeyDB."); } } /* We want to perform a single write. This should be guaranteed atomic @@ -457,7 +477,7 @@ void flushAppendOnlyFile(int force) { if (ftruncate(g_pserver->aof_fd, g_pserver->aof_current_size) == -1) { if (can_log) { serverLog(LL_WARNING, "Could not remove short write " - "from the append-only file. Redis may refuse " + "from the append-only file. KeyDB may refuse " "to load the AOF the next time it starts. " "ftruncate: %s", strerror(errno)); } @@ -471,10 +491,11 @@ void flushAppendOnlyFile(int force) { /* Handle the AOF write error. */ if (g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) { - /* We can't recover when the fsync policy is ALWAYS since the - * reply for the client is already in the output buffers, and we - * have the contract with the user that on acknowledged write data - * is synced on disk. */ + /* We can't recover when the fsync policy is ALWAYS since the reply + * for the client is already in the output buffers (both writes and + * reads), and the changes to the db can't be rolled back. Since we + * have a contract with the user that on acknowledged or observed + * writes are is synced on disk, we must exit. */ serverLog(LL_WARNING,"Can't recover from AOF write error when the AOF fsync policy is 'always'. Exiting..."); exit(1); } else { @@ -496,7 +517,7 @@ void flushAppendOnlyFile(int force) { * OK state and log the event. */ if (g_pserver->aof_last_write_status == C_ERR) { serverLog(LL_WARNING, - "AOF write error looks solved, Redis can write again."); + "AOF write error looks solved, KeyDB can write again."); g_pserver->aof_last_write_status = C_OK; } } @@ -522,7 +543,14 @@ try_fsync: /* redis_fsync is defined as fdatasync() for Linux in order to avoid * flushing metadata. */ latencyStartMonitor(latency); - redis_fsync(g_pserver->aof_fd); /* Let's try to get this data on the disk */ + /* Let's try to get this data on the disk. To guarantee data safe when + * the AOF fsync policy is 'always', we should exit if failed to fsync + * AOF (see comment next to the exit(1) after write error above). */ + if (redis_fsync(g_pserver->aof_fd) == -1) { + serverLog(LL_WARNING,"Can't persist AOF for fsync error when the " + "AOF fsync policy is 'always': %s. Exiting...", strerror(errno)); + exit(1); + } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-fsync-always",latency); g_pserver->aof_fsync_offset = g_pserver->aof_current_size; @@ -594,11 +622,10 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r } decrRefCount(seconds); - argv[0] = createStringObject("PEXPIREAT",9); + argv[0] = shared.pexpireat; argv[1] = key; argv[2] = createStringObjectFromLongLong(when); buf = catAppendOnlyGenericCommand(buf, 3, argv); - decrRefCount(argv[0]); decrRefCount(argv[2]); return buf; } @@ -653,43 +680,40 @@ sds catAppendOnlyExpireMemberAtCommand(sds buf, struct redisCommand *cmd, robj * } sds catCommandForAofAndActiveReplication(sds buf, struct redisCommand *cmd, robj **argv, int argc) -{ - robj *tmpargv[3]; - +{ if (cmd->proc == expireCommand || cmd->proc == pexpireCommand || cmd->proc == expireatCommand) { /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */ buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); - } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) { - /* Translate SETEX/PSETEX to SET and PEXPIREAT */ - tmpargv[0] = createStringObject("SET",3); - tmpargv[1] = argv[1]; - tmpargv[2] = argv[3]; - buf = catAppendOnlyGenericCommand(buf,3,tmpargv); - decrRefCount(tmpargv[0]); - buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); } else if (cmd->proc == setCommand && argc > 3) { - int i; - robj *exarg = NULL, *pxarg = NULL; - for (i = 3; i < argc; i ++) { - if (!strcasecmp(szFromObj(argv[i]), "ex")) exarg = argv[i+1]; - if (!strcasecmp(szFromObj(argv[i]), "px")) pxarg = argv[i+1]; + robj *pxarg = NULL; + /* When SET is used with EX/PX argument setGenericCommand propagates them with PX millisecond argument. + * So since the command arguments are re-written there, we can rely here on the index of PX being 3. */ + if (!strcasecmp(szFromObj(argv[3]), "px")) { + pxarg = argv[4]; } - serverAssert(!(exarg && pxarg)); - if (exarg || pxarg) { - /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */ - buf = catAppendOnlyGenericCommand(buf,3,argv); - if (exarg) - buf = catAppendOnlyExpireAtCommand(buf,cserver.expireCommand,argv[1], - exarg); - if (pxarg) - buf = catAppendOnlyExpireAtCommand(buf,cserver.pexpireCommand,argv[1], - pxarg); + /* For AOF we convert SET key value relative time in milliseconds to SET key value absolute time in + * millisecond. Whenever the condition is true it implies that original SET has been transformed + * to SET PX with millisecond time argument so we do not need to worry about unit here.*/ + if (pxarg) { + robj *millisecond = getDecodedObject(pxarg); + long long when = strtoll(szFromObj(millisecond),NULL,10); + when += mstime(); + + decrRefCount(millisecond); + + robj *newargs[5]; + newargs[0] = argv[0]; + newargs[1] = argv[1]; + newargs[2] = argv[2]; + newargs[3] = shared.pxat; + newargs[4] = createStringObjectFromLongLong(when); + buf = catAppendOnlyGenericCommand(buf,5,newargs); + decrRefCount(newargs[4]); } else { buf = catAppendOnlyGenericCommand(buf,argc,argv); } - } else if (cmd->proc == expireMemberCommand || cmd->proc == expireMemberAtCommand || - cmd->proc == pexpireMemberAtCommand) { + } else if (cmd->proc == expireMemberCommand || cmd->proc == expireMemberAtCommand || cmd->proc == pexpireMemberAtCommand) { /* Translate subkey expire commands to PEXPIREMEMBERAT */ buf = catAppendOnlyExpireMemberAtCommand(buf, cmd, argv, argc); } else { @@ -728,7 +752,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a * accumulate the differences between the child DB and the current one * in a buffer, so that when the child process will do its work we * can append the differences to the new append only file. */ - if (g_pserver->aof_child_pid != -1) + if (g_pserver->child_type == CHILD_TYPE_AOF) aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf)); sdsfree(buf); @@ -752,10 +776,23 @@ struct client *createAOFClient(void) { c->querybuf_peak = 0; c->argc = 0; c->argv = NULL; + c->original_argc = 0; + c->original_argv = NULL; c->bufpos = 0; - c->flags = 0; c->fPendingAsyncWrite = FALSE; c->fPendingAsyncWriteHandler = FALSE; + + /* + * The AOF client should never be blocked (unlike master + * replication connection). + * This is because blocking the AOF client might cause + * deadlock (because potentially no one will unblock it). + * Also, if the AOF client will be blocked just for + * background processing there is a chance that the + * command execution order will be violated. + */ + c->flags = CLIENT_DENY_BLOCKING; + c->btype = BLOCKED_NONE; /* We set the fake client as a replica waiting for the synchronization * so that Redis will not try to send replies to this client. */ @@ -765,8 +802,9 @@ struct client *createAOFClient(void) { c->obuf_soft_limit_reached_time = 0; c->watched_keys = listCreate(); c->peerid = NULL; + c->sockname = NULL; c->resp = 2; - c->puser = NULL; + c->user = NULL; c->mvccCheckpoint = 0; listSetFreeMethod(c->reply,freeClientReplyValue); listSetDupMethod(c->reply,dupClientReplyValue); @@ -790,6 +828,7 @@ void freeFakeClient(struct client *c) { listRelease(c->reply); listRelease(c->watched_keys); freeClientMultiState(c); + freeClientOriginalArgv(c); fastlock_unlock(&c->lock); fastlock_free(&c->lock); delete c; @@ -956,7 +995,7 @@ int loadAppendOnlyFile(char *filename) { fakeClient->cmd = NULL; if (g_pserver->aof_load_truncated) valid_up_to = ftello(fp); if (g_pserver->key_load_delay) - usleep(g_pserver->key_load_delay); + debugDelay(g_pserver->key_load_delay); } /* This point can only be reached when EOF is reached without errors. @@ -1062,15 +1101,25 @@ int rewriteListObject(rio *r, robj *key, robj *o) { if (count == 0) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"RPUSH",5) || + !rioWriteBulkObject(r,key)) + { + quicklistReleaseIterator(li); + return 0; + } } if (entry.value) { - if (rioWriteBulkString(r,(char*)entry.value,entry.sz) == 0) return 0; + if (!rioWriteBulkString(r,(char*)entry.value,entry.sz)) { + quicklistReleaseIterator(li); + return 0; + } } else { - if (rioWriteBulkLongLong(r,entry.longval) == 0) return 0; + if (!rioWriteBulkLongLong(r,entry.longval)) { + quicklistReleaseIterator(li); + return 0; + } } if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; @@ -1096,11 +1145,14 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"SADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"SADD",4) || + !rioWriteBulkObject(r,key)) + { + return 0; + } } - if (rioWriteBulkLongLong(r,llval) == 0) return 0; + if (!rioWriteBulkLongLong(r,llval)) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1114,11 +1166,18 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"SADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"SADD",4) || + !rioWriteBulkObject(r,key)) + { + dictReleaseIterator(di); + return 0; + } + } + if (!rioWriteBulkString(r,ele,sdslen(ele))) { + dictReleaseIterator(di); + return 0; } - if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1155,15 +1214,18 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"ZADD",4) || + !rioWriteBulkObject(r,key)) + { + return 0; + } } - if (rioWriteBulkDouble(r,score) == 0) return 0; + if (!rioWriteBulkDouble(r,score)) return 0; if (vstr != NULL) { - if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0; + if (!rioWriteBulkString(r,(char*)vstr,vlen)) return 0; } else { - if (rioWriteBulkLongLong(r,vll) == 0) return 0; + if (!rioWriteBulkLongLong(r,vll)) return 0; } zzlNext(zl,&eptr,&sptr); if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; @@ -1182,12 +1244,20 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"ZADD",4) || + !rioWriteBulkObject(r,key)) + { + dictReleaseIterator(di); + return 0; + } + } + if (!rioWriteBulkDouble(r,*score) || + !rioWriteBulkString(r,ele,sdslen(ele))) + { + dictReleaseIterator(di); + return 0; } - if (rioWriteBulkDouble(r,*score) == 0) return 0; - if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1236,13 +1306,21 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"HMSET",5) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"HMSET",5) || + !rioWriteBulkObject(r,key)) + { + hashTypeReleaseIterator(hi); + return 0; + } } - if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) == 0) return 0; - if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE) == 0) return 0; + if (!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) || + !rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE)) + { + hashTypeReleaseIterator(hi); + return 0; + } if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1288,6 +1366,20 @@ int rioWriteStreamPendingEntry(rio *r, robj *key, const char *groupname, size_t return 1; } +/* Helper for rewriteStreamObject(): emit the XGROUP CREATECONSUMER is + * needed in order to create consumers that do not have any pending entries. + * All this in the context of the specified key and group. */ +int rioWriteStreamEmptyConsumer(rio *r, robj *key, const char *groupname, size_t groupname_len, streamConsumer *consumer) { + /* XGROUP CREATECONSUMER */ + if (rioWriteBulkCount(r,'*',5) == 0) return 0; + if (rioWriteBulkString(r,"XGROUP",6) == 0) return 0; + if (rioWriteBulkString(r,"CREATECONSUMER",14) == 0) return 0; + if (rioWriteBulkObject(r,key) == 0) return 0; + if (rioWriteBulkString(r,groupname,groupname_len) == 0) return 0; + if (rioWriteBulkString(r,consumer->name,sdslen(consumer->name)) == 0) return 0; + return 1; +} + /* Emit the commands needed to rebuild a stream object. * The function returns 0 on error, 1 on success. */ int rewriteStreamObject(rio *r, robj *key, robj *o) { @@ -1376,13 +1468,25 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { } /* Generate XCLAIMs for each consumer that happens to - * have pending entries. Empty consumers have no semantical - * value so they are discarded. */ + * have pending entries. Empty consumers would be generated with + * XGROUP CREATECONSUMER. */ raxIterator ri_cons; raxStart(&ri_cons,group->consumers); raxSeek(&ri_cons,"^",NULL,0); while(raxNext(&ri_cons)) { streamConsumer *consumer = (streamConsumer*)ri_cons.data; + /* If there are no pending entries, just emit XGROUP CREATECONSUMER */ + if (raxSize(consumer->pel) == 0) { + if (rioWriteStreamEmptyConsumer(r,key,(char*)ri.key, + ri.key_len,consumer) == 0) + { + raxStop(&ri_cons); + raxStop(&ri); + streamIteratorStop(&si); + return 0; + } + continue; + } /* For the current consumer, iterate all the PEL entries * to emit the XCLAIM protocol. */ raxIterator ri_pel; @@ -1446,6 +1550,8 @@ ssize_t aofReadDiffFromParent(void) { int rewriteAppendOnlyFileRio(rio *aof) { size_t processed = 0; int j; + long key_count = 0; + long long updated_time = 0; for (j = 0; j < cserver.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; @@ -1511,6 +1617,18 @@ int rewriteAppendOnlyFileRio(rio *aof) { processed = aof->processed_bytes; aofReadDiffFromParent(); } + + /* Update info every 1 second (approximately). + * in order to avoid calling mstime() on each iteration, we will + * check the diff every 1024 keys */ + if ((key_count++ & 1023) == 0) { + long long now = mstime(); + if (now - updated_time >= 1000) { + sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite"); + updated_time = now; + } + } + return true; }); if (!fComplete) @@ -1537,6 +1655,7 @@ int rewriteAppendOnlyFile(char *filename) { int nodata = 0; mstime_t start = 0; +{ // BEGIN GOTO SCOPED VARIABLES /* Note that we have to use a different temp name here compared to the * one used by rewriteAppendOnlyFileBackground() function. */ snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid()); @@ -1596,7 +1715,7 @@ int rewriteAppendOnlyFile(char *filename) { if (write(g_pserver->aof_pipe_write_ack_to_parent,"!",1) != 1) goto werr; if (anetNonBlock(NULL,g_pserver->aof_pipe_read_ack_from_parent) != ANET_OK) goto werr; - /* We read the ACK from the server using a 10 seconds timeout. Normally + /* We read the ACK from the server using a 5 seconds timeout. Normally * it should reply ASAP, but just in case we lose its reply, we are sure * the child will eventually get terminated. */ if (syncRead(g_pserver->aof_pipe_read_ack_from_parent,&byte,1,5000) != 1 || @@ -1610,8 +1729,31 @@ int rewriteAppendOnlyFile(char *filename) { serverLog(LL_NOTICE, "Concatenating %.2f MB of AOF diff received from parent.", (double) sdslen(g_pserver->aof_child_diff) / (1024*1024)); - if (rioWrite(&aof,g_pserver->aof_child_diff,sdslen(g_pserver->aof_child_diff)) == 0) - goto werr; + + /* Now we write the entire AOF buffer we received from the parent + * via the pipe during the life of this fork child. + * once a second, we'll take a break and send updated COW info to the parent */ + size_t bytes_to_write = sdslen(g_pserver->aof_child_diff); + const char *buf = g_pserver->aof_child_diff; + long long cow_updated_time = mstime(); + long long key_count = dbTotalServerKeyCount(); + while (bytes_to_write) { + /* We write the AOF buffer in chunk of 8MB so that we can check the time in between them */ + size_t chunk_size = bytes_to_write < (8<<20) ? bytes_to_write : (8<<20); + + if (rioWrite(&aof,buf,chunk_size) == 0) + goto werr; + + bytes_to_write -= chunk_size; + buf += chunk_size; + + /* Update COW info */ + long long now = mstime(); + if (now - cow_updated_time >= 1000) { + sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite"); + cow_updated_time = now; + } + } /* Make sure data will not remain on the OS's output buffers */ if (fflush(fp)) goto werr; @@ -1630,6 +1772,7 @@ int rewriteAppendOnlyFile(char *filename) { serverLog(LL_NOTICE,"SYNC append only file rewrite performed"); stopSaving(1); return C_OK; +} // END GOTO SCOPED VARIABLES werr: serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); @@ -1746,7 +1889,6 @@ int rewriteAppendOnlyFileBackground(void) { if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; - openChildInfoPipe(); if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) { char tmpfile[256]; @@ -1755,7 +1897,7 @@ int rewriteAppendOnlyFileBackground(void) { redisSetCpuAffinity(g_pserver->aof_rewrite_cpulist); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - sendChildCOWInfo(CHILD_TYPE_AOF, "AOF rewrite"); + sendChildCowInfo(CHILD_INFO_TYPE_AOF_COW_SIZE, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); @@ -1763,7 +1905,6 @@ int rewriteAppendOnlyFileBackground(void) { } else { /* Parent */ if (childpid == -1) { - closeChildInfoPipe(); serverLog(LL_WARNING, "Can't rewrite append only file in background: fork: %s", strerror(errno)); @@ -1771,10 +1912,9 @@ int rewriteAppendOnlyFileBackground(void) { return C_ERR; } serverLog(LL_NOTICE, - "Background append only file rewriting started by pid %d",childpid); + "Background append only file rewriting started by pid %ld",(long)childpid); g_pserver->aof_rewrite_scheduled = 0; g_pserver->aof_rewrite_time_start = time(NULL); - g_pserver->aof_child_pid = childpid; updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences @@ -1788,7 +1928,7 @@ int rewriteAppendOnlyFileBackground(void) { } void bgrewriteaofCommand(client *c) { - if (g_pserver->aof_child_pid != -1) { + if (g_pserver->child_type == CHILD_TYPE_AOF) { addReplyError(c,"Background append only file rewriting already in progress"); } else if (hasActiveChildProcess()) { g_pserver->aof_rewrite_scheduled = 1; @@ -1846,7 +1986,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { * rewritten AOF. */ latencyStartMonitor(latency); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", - (int)g_pserver->aof_child_pid); + (int)g_pserver->child_pid); newfd = open(tmpfile,O_WRONLY|O_APPEND); if (newfd == -1) { serverLog(LL_WARNING, @@ -1862,6 +2002,20 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency); + + if (g_pserver->aof_fsync == AOF_FSYNC_EVERYSEC) { + aof_background_fsync(newfd); + } else if (g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) { + latencyStartMonitor(latency); + if (redis_fsync(newfd) == -1) { + serverLog(LL_WARNING, + "Error trying to fsync the parent diff to the rewritten AOF: %s", strerror(errno)); + close(newfd); + goto cleanup; + } + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-rewrite-done-fsync",latency); + } serverLog(LL_NOTICE, "Residual parent diff successfully flushed to the rewritten AOF (%.2f MB)", (double) aofRewriteBufferSize() / (1024*1024)); @@ -1929,14 +2083,11 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { /* AOF enabled, replace the old fd with the new one. */ oldfd = g_pserver->aof_fd; g_pserver->aof_fd = newfd; - if (g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) - redis_fsync(newfd); - else if (g_pserver->aof_fsync == AOF_FSYNC_EVERYSEC) - aof_background_fsync(newfd); g_pserver->aof_selected_db = -1; /* Make sure SELECT is re-issued */ aofUpdateCurrentSize(); g_pserver->aof_rewrite_base_size = g_pserver->aof_current_size; g_pserver->aof_fsync_offset = g_pserver->aof_current_size; + g_pserver->aof_last_fsync = g_pserver->unixtime; /* Clear regular AOF buffer since its contents was just written to * the new AOF from the background rewrite buffer. */ @@ -1952,7 +2103,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { g_pserver->aof_state = AOF_ON; /* Asynchronously close the overwritten AOF. */ - if (oldfd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL); + if (oldfd != -1) bioCreateCloseJob(oldfd); serverLog(LL_VERBOSE, "Background AOF rewrite signal handler took %lldus", ustime()-now); @@ -1974,8 +2125,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { cleanup: aofClosePipes(); aofRewriteBufferReset(); - aofRemoveTempFile(g_pserver->aof_child_pid); - g_pserver->aof_child_pid = -1; + aofRemoveTempFile(g_pserver->child_pid); g_pserver->aof_rewrite_time_last = time(NULL)-g_pserver->aof_rewrite_time_start; g_pserver->aof_rewrite_time_start = -1; /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */ diff --git a/src/asciilogo.h b/src/asciilogo.h index d4dfe6a77..f4fbd360e 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -41,7 +41,7 @@ const char *ascii_logo = " | / | \\ | https://docs.keydb.dev \n" " | / | \\ | \n" " | / | \\ | \n" -" (+)_ -- -- -- | -- -- -- _(+) \n" +" (+)_ -- -- -- | -- -- -- _(+) \n" " --_ | _-- \n" " --_ | _-- \n" " -(+)- %s\n" diff --git a/src/atomicvar.h b/src/atomicvar.h index ecd26ad70..222b82699 100644 --- a/src/atomicvar.h +++ b/src/atomicvar.h @@ -1,5 +1,5 @@ -/* This file implements atomic counters using __atomic or __sync macros if - * available, otherwise synchronizing different threads using a mutex. +/* This file implements atomic counters using c11 _Atomic, __atomic or __sync + * macros if available, otherwise we will throw an error when compile. * * The exported interface is composed of three macros: * @@ -8,16 +8,8 @@ * atomicDecr(var,count) -- Decrement the atomic counter * atomicGet(var,dstvar) -- Fetch the atomic counter value * atomicSet(var,value) -- Set the atomic counter value - * - * The variable 'var' should also have a declared mutex with the same - * name and the "_mutex" postfix, for instance: - * - * long myvar; - * pthread_mutex_t myvar_mutex; - * atomicSet(myvar,12345); - * - * If atomic primitives are available (tested in config.h) the mutex - * is not used. + * atomicGetWithSync(var,value) -- 'atomicGet' with inter-thread synchronization + * atomicSetWithSync(var,value) -- 'atomicSet' with inter-thread synchronization * * Never use return value from the macros, instead use the AtomicGetIncr() * if you need to get the current value and increment it atomically, like @@ -58,17 +50,64 @@ */ #include +#include "config.h" #ifndef __ATOMIC_VAR_H #define __ATOMIC_VAR_H +/* Define redisAtomic for atomic variable. */ +#define redisAtomic + /* To test Redis with Helgrind (a Valgrind tool) it is useful to define * the following macro, so that __sync macros are used: those can be detected * by Helgrind (even if they are less efficient) so that no false positive * is reported. */ // #define __ATOMIC_VAR_FORCE_SYNC_MACROS -#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__ATOMIC_RELAXED) && !defined(__sun) && (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) +/* There will be many false positives if we test Redis with Helgrind, since + * Helgrind can't understand we have imposed ordering on the program, so + * we use macros in helgrind.h to tell Helgrind inter-thread happens-before + * relationship explicitly for avoiding false positives. + * + * For more details, please see: valgrind/helgrind.h and + * https://www.valgrind.org/docs/manual/hg-manual.html#hg-manual.effective-use + * + * These macros take effect only when 'make helgrind', and you must first + * install Valgrind in the default path configuration. */ +#ifdef __ATOMIC_VAR_FORCE_SYNC_MACROS +#include +#else +#define ANNOTATE_HAPPENS_BEFORE(v) ((void) v) +#define ANNOTATE_HAPPENS_AFTER(v) ((void) v) +#endif + +#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) +/* Use '_Atomic' keyword if the compiler supports. */ +#undef redisAtomic +#define redisAtomic _Atomic +/* Implementation using _Atomic in C11. */ + +#include +#define atomicIncr(var,count) atomic_fetch_add_explicit(&var,(count),memory_order_relaxed) +#define atomicGetIncr(var,oldvalue_var,count) do { \ + oldvalue_var = atomic_fetch_add_explicit(&var,(count),memory_order_relaxed); \ +} while(0) +#define atomicDecr(var,count) atomic_fetch_sub_explicit(&var,(count),memory_order_relaxed) +#define atomicGet(var,dstvar) do { \ + dstvar = atomic_load_explicit(&var,memory_order_relaxed); \ +} while(0) +#define atomicSet(var,value) atomic_store_explicit(&var,value,memory_order_relaxed) +#define atomicGetWithSync(var,dstvar) do { \ + dstvar = atomic_load_explicit(&var,memory_order_seq_cst); \ +} while(0) +#define atomicSetWithSync(var,value) \ + atomic_store_explicit(&var,value,memory_order_seq_cst) +#define REDIS_ATOMIC_API "c11-builtin" + +#elif !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && \ + (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) && \ + defined(__ATOMIC_RELAXED) && defined(__ATOMIC_SEQ_CST) /* Implementation using __atomic macros. */ #define atomicIncr(var,count) __atomic_add_fetch(&var,(count),__ATOMIC_RELAXED) @@ -80,6 +119,11 @@ dstvar = __atomic_load_n(&var,__ATOMIC_RELAXED); \ } while(0) #define atomicSet(var,value) __atomic_store_n(&var,value,__ATOMIC_RELAXED) +#define atomicGetWithSync(var,dstvar) do { \ + dstvar = __atomic_load_n(&var,__ATOMIC_SEQ_CST); \ +} while(0) +#define atomicSetWithSync(var,value) \ + __atomic_store_n(&var,value,__ATOMIC_SEQ_CST) #define REDIS_ATOMIC_API "atomic-builtin" #elif defined(HAVE_ATOMIC) @@ -96,38 +140,19 @@ #define atomicSet(var,value) do { \ while(!__sync_bool_compare_and_swap(&var,var,value)); \ } while(0) +/* Actually the builtin issues a full memory barrier by default. */ +#define atomicGetWithSync(var,dstvar) { \ + dstvar = __sync_sub_and_fetch(&var,0,__sync_synchronize); \ + ANNOTATE_HAPPENS_AFTER(&var); \ +} while(0) +#define atomicSetWithSync(var,value) do { \ + ANNOTATE_HAPPENS_BEFORE(&var); \ + while(!__sync_bool_compare_and_swap(&var,var,value,__sync_synchronize)); \ +} while(0) #define REDIS_ATOMIC_API "sync-builtin" #else -/* Implementation using pthread mutex. */ - -#define atomicIncr(var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var += (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicGetIncr(var,oldvalue_var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - oldvalue_var = var; \ - var += (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicDecr(var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var -= (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicGet(var,dstvar) do { \ - pthread_mutex_lock(&var ## _mutex); \ - dstvar = var; \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicSet(var,value) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var = value; \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define REDIS_ATOMIC_API "pthread-mutex" +#error "Unable to determine atomic operations for your platform" #endif #endif /* __ATOMIC_VAR_H */ diff --git a/src/bio.cpp b/src/bio.cpp index 2c28e57a0..d2baa23e4 100644 --- a/src/bio.cpp +++ b/src/bio.cpp @@ -78,15 +78,14 @@ static unsigned long long bio_pending[BIO_NUM_OPS]; * file as the API does not expose the internals at all. */ struct bio_job { time_t time; /* Time at which the job was created. */ - /* Job specific arguments pointers. If we need to pass more than three - * arguments we can just pass a pointer to a structure or alike. */ - void *arg1, *arg2, *arg3; + /* Job specific arguments.*/ + int fd; /* Fd for file based background jobs */ + lazy_free_fn *free_fn; /* Function that will free the provided arguments */ + void ** free_args() { return reinterpret_cast(this+1); } /* List of arguments to be passed to the free function */ + void set_free_arg(int i, void *arg) { reinterpret_cast(this+1)[i] = arg; } }; void *bioProcessBackgroundJobs(void *arg); -void lazyfreeFreeObjectFromBioThread(robj *o); -void lazyfreeFreeDatabaseFromBioThread(dict *ht1, expireset *set); -void lazyfreeFreeSlotsMapFromBioThread(rax *rt); /* Make sure we have enough stack to perform all the things we do in the * main thread. */ @@ -128,13 +127,8 @@ void bioInit(void) { } } -void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { - struct bio_job *job = (bio_job*)zmalloc(sizeof(*job), MALLOC_LOCAL); - +void bioSubmitJob(int type, struct bio_job *job) { job->time = time(NULL); - job->arg1 = arg1; - job->arg2 = arg2; - job->arg3 = arg3; pthread_mutex_lock(&bio_mutex[type]); listAddNodeTail(bio_jobs[type],job); bio_pending[type]++; @@ -142,6 +136,35 @@ void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { pthread_mutex_unlock(&bio_mutex[type]); } +void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) { + va_list valist; + /* Allocate memory for the job structure and all required + * arguments */ + struct bio_job *job = (bio_job*)zmalloc(sizeof(*job) + sizeof(void *) * (arg_count)); + job->free_fn = free_fn; + + va_start(valist, arg_count); + for (int i = 0; i < arg_count; i++) { + job->set_free_arg(i, va_arg(valist, void *)); + } + va_end(valist); + bioSubmitJob(BIO_LAZY_FREE, job); +} + +void bioCreateCloseJob(int fd) { + struct bio_job *job = (bio_job*)zmalloc(sizeof(*job)); + job->fd = fd; + + bioSubmitJob(BIO_CLOSE_FILE, job); +} + +void bioCreateFsyncJob(int fd) { + struct bio_job *job = (bio_job*)zmalloc(sizeof(*job)); + job->fd = fd; + + bioSubmitJob(BIO_AOF_FSYNC, job); +} + void *bioProcessBackgroundJobs(void *arg) { struct bio_job *job; unsigned long type = (unsigned long) arg; @@ -196,20 +219,27 @@ void *bioProcessBackgroundJobs(void *arg) { /* Process the job accordingly to its type. */ if (type == BIO_CLOSE_FILE) { - close((long)job->arg1); + close(job->fd); } else if (type == BIO_AOF_FSYNC) { - redis_fsync((long)job->arg1); + /* The fd may be closed by main thread and reused for another + * socket, pipe, or file. We just ignore these errno because + * aof fsync did not really fail. */ + if (redis_fsync(job->fd) == -1 && + errno != EBADF && errno != EINVAL) + { + int last_status; + atomicGet(g_pserver->aof_bio_fsync_status,last_status); + atomicSet(g_pserver->aof_bio_fsync_status,C_ERR); + atomicSet(g_pserver->aof_bio_fsync_errno,errno); + if (last_status == C_OK) { + serverLog(LL_WARNING, + "Fail to fsync the AOF file: %s",strerror(errno)); + } + } else { + atomicSet(g_pserver->aof_bio_fsync_status,C_OK); + } } else if (type == BIO_LAZY_FREE) { - /* What we free changes depending on what arguments are set: - * arg1 -> free the object at pointer. - * arg2 & arg3 -> free two dictionaries (a Redis DB). - * only arg3 -> free the radix tree. */ - if (job->arg1) - lazyfreeFreeObjectFromBioThread((robj*)job->arg1); - else if (job->arg2 && job->arg3) - lazyfreeFreeDatabaseFromBioThread((dict*)job->arg2,(expireset*)job->arg3); - else if (job->arg3) - lazyfreeFreeSlotsMapFromBioThread((rax*)job->arg3); + job->free_fn(job->free_args()); } else { serverPanic("Wrong job type in bioProcessBackgroundJobs()."); } diff --git a/src/bio.h b/src/bio.h index bf350f9de..b0ce9bbad 100644 --- a/src/bio.h +++ b/src/bio.h @@ -32,13 +32,17 @@ extern "C" { #endif +typedef void lazy_free_fn(void *args[]); + /* Exported API */ void bioInit(void); -void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3); unsigned long long bioPendingJobsOfType(int type); unsigned long long bioWaitStepOfType(int type); time_t bioOlderJobOfType(int type); void bioKillThreads(void); +void bioCreateCloseJob(int fd); +void bioCreateFsyncJob(int fd); +void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...); /* Background job opcodes */ #define BIO_CLOSE_FILE 0 /* Deferred close(2) syscall. */ diff --git a/src/bitops.cpp b/src/bitops.cpp index 8b26114db..c03f068fd 100644 --- a/src/bitops.cpp +++ b/src/bitops.cpp @@ -36,7 +36,7 @@ /* Count number of bits set in the binary array pointed by 's' and long * 'count' bytes. The implementation of this function is required to - * work with an input string length up to 512 MB. */ + * work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */ size_t redisPopcount(const void *s, long count) { size_t bits = 0; unsigned char *p = (unsigned char*)s; @@ -407,7 +407,7 @@ void printBits(unsigned char *p, unsigned long count) { /* This helper function used by GETBIT / SETBIT parses the bit offset argument * making sure an error is returned if it is negative or if it overflows - * Redis 512 MB limit for the string value. + * Redis 512 MB limit for the string value or more (server.proto_max_bulk_len). * * If the 'hash' argument is true, and 'bits is positive, then the command * will also parse bit offsets prefixed by "#". In such a case the offset @@ -430,8 +430,8 @@ int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int b /* Adjust the offset by 'bits' for # form. */ if (usehash) loffset *= bits; - /* Limit offset to 512MB in bytes */ - if ((loffset < 0) || ((unsigned long long)loffset >> 3) >= (512*1024*1024)) + /* Limit offset to server.proto_max_bulk_len (512MB in bytes by default) */ + if ((loffset < 0) || (loffset >> 3) >= g_pserver->proto_max_bulk_len) { addReplyError(c,err); return C_ERR; @@ -482,12 +482,12 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) { robj *lookupStringForBitCommand(client *c, size_t maxbit) { size_t byte = maxbit >> 3; robj *o = lookupKeyWrite(c->db,c->argv[1]); + if (checkType(c,o,OBJ_STRING)) return NULL; if (o == NULL) { o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1)); dbAdd(c->db,c->argv[1],o); } else { - if (checkType(c,o,OBJ_STRING)) return NULL; o = dbUnshareStringValue(c->db,c->argv[1],o); o->m_ptr = sdsgrowzero(szFromObj(o),byte+1); } @@ -619,7 +619,7 @@ void bitopCommand(client *c) { else if (!strcasecmp(opname, "rshift")) op = BITOP_RSHIFT; else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -731,7 +731,6 @@ void bitopCommand(client *c) { /* Compute the bit operation, if at least one string is not empty. */ if (maxlen) { res = (unsigned char*) sdsnewlen(NULL,maxlen); - unsigned char output, byte; unsigned long i; /* Fast path: as far as we have data for all the input bitmaps we @@ -801,22 +800,33 @@ void bitopCommand(client *c) { } } } - #endif + } + #endif - /* j is set to the next byte to process by the previous loop. */ - for (; j < maxlen; j++) { - output = (len[0] <= j) ? 0 : src[0][j]; - if (op == BITOP_NOT) output = ~output; - for (i = 1; i < numkeys; i++) { - byte = (len[i] <= j) ? 0 : src[i][j]; - switch(op) { - case BITOP_AND: output &= byte; break; - case BITOP_OR: output |= byte; break; - case BITOP_XOR: output ^= byte; break; - } + /* j is set to the next byte to process by the previous loop. */ + for (; j < maxlen; j++) { + auto output = (len[0] <= j) ? 0 : src[0][j]; + if (op == BITOP_NOT) output = ~output; + for (unsigned long i = 1; i < numkeys; i++) { + int skip = 0; + auto byte = (len[i] <= j) ? 0 : src[i][j]; + switch(op) { + case BITOP_AND: + output &= byte; + skip = (output == 0); + break; + case BITOP_OR: + output |= byte; + skip = (output == 0xff); + break; + case BITOP_XOR: output ^= byte; break; + } + + if (skip) { + break; } - res[j] = output; } + res[j] = output; } } for (j = 0; j < numkeys; j++) { @@ -876,7 +886,7 @@ void bitcountCommand(client *c) { end = strlen-1; } else { /* Syntax error. */ - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -941,7 +951,7 @@ void bitposCommand(client *c) { end = strlen-1; } else { /* Syntax error. */ - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -1033,7 +1043,7 @@ void bitfieldGeneric(client *c, int flags) { } continue; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); zfree(ops); return; } diff --git a/src/blocked.cpp b/src/blocked.cpp index a37dcca58..0d024081f 100644 --- a/src/blocked.cpp +++ b/src/blocked.cpp @@ -61,9 +61,13 @@ */ #include "server.h" +#include "slowlog.h" +#include "latency.h" +#include "monotonic.h" #include -int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where); +int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int wherefrom, int whereto); +int getListPositionFromObjectOrReply(client *c, robj *arg, int *position); /* This structure represents the blocked key information that we store * in the client structure. Each client blocked on keys, has a @@ -92,6 +96,25 @@ void blockClient(client *c, int btype) { g_pserver->blocked_clients++; g_pserver->blocked_clients_by_type[btype]++; addClientToTimeoutTable(c); + if (btype == BLOCKED_PAUSE) { + listAddNodeTail(g_pserver->paused_clients, c); + c->paused_list_node = listLast(g_pserver->paused_clients); + /* Mark this client to execute its command */ + c->flags |= CLIENT_PENDING_COMMAND; + } +} + +/* This function is called after a client has finished a blocking operation + * in order to update the total command duration, log the command into + * the Slow log if needed, and log the reply duration event if needed. */ +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ + const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; + c->lastcmd->microseconds += total_cmd_duration; + + /* Log the command into the Slow log if needed. */ + slowlogPushCurrentCommand(c, c->lastcmd, total_cmd_duration); + /* Log the reply duration event. */ + latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); } /* This function is called in the beforeSleep() function of the event loop @@ -112,7 +135,7 @@ void processUnblockedClients(int iel) { listDelNode(unblocked_clients,ln); AssertCorrectThread(c); - fastlock_lock(&c->lock); + std::unique_lock ul(c->lock); c->flags &= ~CLIENT_UNBLOCKED; /* Process remaining data in the input buffer, unless the client @@ -120,11 +143,15 @@ void processUnblockedClients(int iel) { * client is not blocked before to proceed, but things may change and * the code is conceptually more correct this way. */ if (!(c->flags & CLIENT_BLOCKED)) { + /* If we have a queued command, execute it now. */ + if (processPendingCommandsAndResetClient(c, CMD_CALL_FULL) == C_ERR) { + continue; + } + /* Then process client if it has more data in it's buffer. */ if (c->querybuf && sdslen(c->querybuf) > 0) { processInputBuffer(c, true /*fParse*/, CMD_CALL_FULL); } } - fastlock_unlock(&c->lock); } } @@ -173,9 +200,22 @@ void unblockClient(client *c) { } else if (c->btype == BLOCKED_ASYNC) { serverAssert(c->casyncOpsPending > 0); c->casyncOpsPending--; + } else if (c->btype == BLOCKED_PAUSE) { + listDelNode(g_pserver->paused_clients,c->paused_list_node); + c->paused_list_node = NULL; } else { serverPanic("Unknown btype in unblockClient()."); } + + /* Reset the client for a new query since, for blocking commands + * we do not do it immediately after the command returns (when the + * client got blocked) in order to be still able to access the argument + * vector from module callbacks and updateStatsOnUnblock. */ + if (c->btype != BLOCKED_PAUSE) { + freeClientOriginalArgv(c); + resetClient(c); + } + /* Clear the flags, and put the client in the unblocked list so that * we'll process new commands in its query buffer ASAP. */ g_pserver->blocked_clients--; @@ -219,15 +259,21 @@ void disconnectAllBlockedClients(void) { while((ln = listNext(&li))) { client *c = (client*)listNodeValue(ln); - fastlock_lock(&c->lock); + std::unique_lock ul(c->lock); if (c->flags & CLIENT_BLOCKED) { - addReplySds(c,sdsnew( + /* PAUSED clients are an exception, when they'll be unblocked, the + * command processing will start from scratch, and the command will + * be either executed or rejected. (unlike LIST blocked clients for + * which the command is already in progress in a way. */ + if (c->btype == BLOCKED_PAUSE) + continue; + + addReplyError(c, "-UNBLOCKED force unblock from blocking operation, " - "instance state changed (master -> replica?)\r\n")); + "instance state changed (master -> replica?)"); unblockClient(c); c->flags |= CLIENT_CLOSE_AFTER_REPLY; } - fastlock_unlock(&c->lock); } } @@ -255,26 +301,28 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { } robj *dstkey = receiver->bpop.target; - int where = (receiver->lastcmd && - receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; - robj *value = listTypePop(o,where); + int wherefrom = receiver->bpop.listpos.wherefrom; + int whereto = receiver->bpop.listpos.whereto; + robj *value = listTypePop(o, wherefrom); if (value) { /* Protect receiver->bpop.target, that will be * freed by the next unblockClient() * call. */ if (dstkey) incrRefCount(dstkey); - unblockClient(receiver); + monotime replyTimer; + elapsedStart(&replyTimer); if (serveClientBlockedOnList(receiver, rl->key,dstkey,rl->db,value, - where) == C_ERR) + wherefrom, whereto) == C_ERR) { /* If we failed serving the client we need * to also undo the POP operation. */ - listTypePush(o,value,where); + listTypePush(o,value,wherefrom); } + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); if (dstkey) decrRefCount(dstkey); decrRefCount(value); @@ -319,8 +367,11 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { int where = (receiver->lastcmd && receiver->lastcmd->proc == bzpopminCommand) ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); + monotime replyTimer; + elapsedStart(&replyTimer); genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); zcard--; /* Replicate the command. */ @@ -397,13 +448,22 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { int noack = 0; if (group) { + int created = 0; consumer = streamLookupConsumer(group, szFromObj(receiver->bpop.xread_consumer), - SLC_NONE); + SLC_NONE, + &created); noack = receiver->bpop.xread_group_noack; + if (created && noack) { + streamPropagateConsumerCreation(receiver,rl->key, + receiver->bpop.xread_group, + consumer->name); + } } + monotime replyTimer; + elapsedStart(&replyTimer); /* Emit the two elements sub-array consisting of * the name of the stream and the data we * extracted from it. Wrapped in a single-item @@ -423,6 +483,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { streamReplyWithRange(receiver,s,&start,NULL, receiver->bpop.xread_count, 0, group, consumer, noack, &pi); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); /* Note that after we unblock the client, 'gt' * and other receiver->bpop stuff are no longer @@ -443,6 +504,10 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { void serveClientsBlockedOnKeyByModule(readyList *rl) { dictEntry *de; + /* Optimization: If no clients are in type BLOCKED_MODULE, + * we can skip this loop. */ + if (!g_pserver->blocked_clients_by_type[BLOCKED_MODULE]) return; + /* We serve clients in the same order they blocked for * this key, from the first blocked to the last. */ de = dictFind(rl->db->blocking_keys,rl->key); @@ -469,7 +534,10 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * different modules with different triggers to consider if a key * is ready or not. This means we can't exit the loop but need * to continue after the first failure. */ + monotime replyTimer; + elapsedStart(&replyTimer); if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); moduleUnblockClient(receiver); } @@ -485,8 +553,8 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * one new element via some write operation are accumulated into * the g_pserver->ready_keys list. This function will run the list and will * serve clients accordingly. Note that the function will iterate again and - * again as a result of serving BRPOPLPUSH we can have new blocking clients - * to serve because of the PUSH side of BRPOPLPUSH. + * again as a result of serving BLMOVE we can have new blocking clients + * to serve because of the PUSH side of BLMOVE. * * This function is normally "fair", that is, it will server clients * using a FIFO behavior. However this fairness is violated in certain @@ -520,10 +588,10 @@ void handleClientsBlockedOnKeys(void) { /* Even if we are not inside call(), increment the call depth * in order to make sure that keys are expired against a fixed * reference time, and not against the wallclock time. This - * way we can lookup an object multiple times (BRPOPLPUSH does + * way we can lookup an object multiple times (BLMOVE does * that) without the risk of it being freed in the second * lookup, invalidating the first one. - * See https://github.com/antirez/redis/pull/6554. */ + * See https://github.com/redis/redis/pull/6554. */ serverTL->fixed_time_expire++; /* Serve clients blocked on the key. */ @@ -579,7 +647,7 @@ void handleClientsBlockedOnKeys(void) { * stream keys, we also provide an array of streamID structures: clients will * be unblocked only when items with an ID greater or equal to the specified * one is appended to the stream. */ -void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids) { +void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, struct listPos *listpos, streamID *ids) { dictEntry *de; list *l; int j; @@ -587,6 +655,8 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo c->bpop.timeout = timeout; c->bpop.target = target; + if (listpos != NULL) c->bpop.listpos = *listpos; + if (target != NULL) incrRefCount(target); for (j = 0; j < numkeys; j++) { @@ -660,6 +730,16 @@ void unblockClientWaitingData(client *c) { } } +static int getBlockedTypeByType(int type) { + switch (type) { + case OBJ_LIST: return BLOCKED_LIST; + case OBJ_ZSET: return BLOCKED_ZSET; + case OBJ_MODULE: return BLOCKED_MODULE; + case OBJ_STREAM: return BLOCKED_STREAM; + default: return BLOCKED_NONE; + } +} + /* If the specified key has clients blocked waiting for list pushes, this * function will put the key reference into the g_pserver->ready_keys list. * Note that db->ready_keys is a hash table that allows us to avoid putting @@ -667,9 +747,24 @@ void unblockClientWaitingData(client *c) { * made by a script or in the context of MULTI/EXEC. * * The list will be finally processed by handleClientsBlockedOnKeys() */ -void signalKeyAsReady(redisDb *db, robj *key) { +void signalKeyAsReady(redisDb *db, robj *key, int type) { readyList *rl; + /* Quick returns. */ + int btype = getBlockedTypeByType(type); + if (btype == BLOCKED_NONE) { + /* The type can never block. */ + return; + } + if (!g_pserver->blocked_clients_by_type[btype] && + !g_pserver->blocked_clients_by_type[BLOCKED_MODULE]) { + /* No clients block on this type. Note: Blocked modules are represented + * by BLOCKED_MODULE, even if the intention is to wake up by normal + * types (list, zset, stream), so we need to check that there are no + * blocked modules before we do a quick return here. */ + return; + } + /* No clients blocking for this key? No need to queue it. */ if (dictFind(db->blocking_keys,key) == NULL) return; @@ -697,4 +792,8 @@ void signalKeyAsReady(redisDb *db, robj *key) { serverAssert(dictAdd(db->ready_keys,key,NULL) == DICT_OK); } - +void signalKeyAsReady(redisDb *db, sds key, int type) { + redisObjectStack o; + initStaticStringObject(o, key); + signalKeyAsReady(db, &o, type); +} \ No newline at end of file diff --git a/src/childinfo.cpp b/src/childinfo.cpp index 77900ac11..6ee28ff58 100644 --- a/src/childinfo.cpp +++ b/src/childinfo.cpp @@ -30,18 +30,26 @@ #include "server.h" #include +typedef struct { + size_t keys; + size_t cow; + monotime cow_updated; + double progress; + childInfoType information_type; /* Type of information */ +} child_info_data; + /* Open a child-parent channel used in order to move information about the * RDB / AOF saving process from the child to the parent (for instance * the amount of copy on write memory used) */ void openChildInfoPipe(void) { if (pipe(g_pserver->child_info_pipe) == -1) { /* On error our two file descriptors should be still set to -1, - * but we call anyway cloesChildInfoPipe() since can't hurt. */ + * but we call anyway closeChildInfoPipe() since can't hurt. */ closeChildInfoPipe(); } else if (anetNonBlock(NULL,g_pserver->child_info_pipe[0]) != ANET_OK) { closeChildInfoPipe(); } else { - memset(&g_pserver->child_info_data,0,sizeof(g_pserver->child_info_data)); + g_pserver->child_info_nread = 0; } } @@ -54,34 +62,113 @@ void closeChildInfoPipe(void) { close(g_pserver->child_info_pipe[1]); g_pserver->child_info_pipe[0] = -1; g_pserver->child_info_pipe[1] = -1; + g_pserver->child_info_nread = 0; } } -/* Send COW data to parent. The child should call this function after populating - * the corresponding fields it want to sent (according to the process type). */ -void sendChildInfo(int ptype) { +/* Send save data to parent. */ +void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, const char *pname) { if (g_pserver->child_info_pipe[1] == -1) return; - g_pserver->child_info_data.magic = CHILD_INFO_MAGIC; - g_pserver->child_info_data.process_type = ptype; - ssize_t wlen = sizeof(g_pserver->child_info_data); - if (write(g_pserver->child_info_pipe[1],&g_pserver->child_info_data,wlen) != wlen) { + if (g_pserver->rdbThreadVars.fRdbThreadActive && g_pserver->rdbThreadVars.fRdbThreadCancel) return; + + static monotime cow_updated = 0; + static uint64_t cow_update_cost = 0; + static size_t cow = 0; + + child_info_data data = {0}; /* zero everything, including padding to satisfy valgrind */ + + /* When called to report current info, we need to throttle down CoW updates as they + * can be very expensive. To do that, we measure the time it takes to get a reading + * and schedule the next reading to happen not before time*CHILD_COW_COST_FACTOR + * passes. */ + + monotime now = getMonotonicUs(); + if (info_type != CHILD_INFO_TYPE_CURRENT_INFO || + !cow_updated || + now - cow_updated > cow_update_cost * CHILD_COW_DUTY_CYCLE) + { + cow = zmalloc_get_private_dirty(-1); + cow_updated = getMonotonicUs(); + cow_update_cost = cow_updated - now; + + if (cow) { + serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, data.cow / (1024 * 1024)); + } + } + + data.information_type = info_type; + data.keys = keys; + data.cow = cow; + data.cow_updated = cow_updated; + data.progress = progress; + + ssize_t wlen = sizeof(data); + + if (write(g_pserver->child_info_pipe[1], &data, wlen) != wlen) { /* Nothing to do on error, this will be detected by the other side. */ } } -/* Receive COW data from parent. */ -void receiveChildInfo(void) { - if (g_pserver->child_info_pipe[0] == -1) return; - ssize_t wlen = sizeof(g_pserver->child_info_data); - if (read(g_pserver->child_info_pipe[0],&g_pserver->child_info_data,wlen) == wlen && - g_pserver->child_info_data.magic == CHILD_INFO_MAGIC) - { - if (g_pserver->child_info_data.process_type == CHILD_TYPE_RDB) { - g_pserver->stat_rdb_cow_bytes = g_pserver->child_info_data.cow_size; - } else if (g_pserver->child_info_data.process_type == CHILD_TYPE_AOF) { - g_pserver->stat_aof_cow_bytes = g_pserver->child_info_data.cow_size; - } else if (g_pserver->child_info_data.process_type == CHILD_TYPE_MODULE) { - g_pserver->stat_module_cow_bytes = g_pserver->child_info_data.cow_size; - } +/* Update Child info. */ +void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_updated, size_t keys, double progress) { + if (information_type == CHILD_INFO_TYPE_CURRENT_INFO) { + g_pserver->stat_current_cow_bytes = cow; + g_pserver->stat_current_cow_updated = cow_updated; + g_pserver->stat_current_save_keys_processed = keys; + if (progress != -1) g_pserver->stat_module_progress = progress; + } else if (information_type == CHILD_INFO_TYPE_AOF_COW_SIZE) { + g_pserver->stat_aof_cow_bytes = cow; + } else if (information_type == CHILD_INFO_TYPE_RDB_COW_SIZE) { + g_pserver->stat_rdb_cow_bytes = cow; + } else if (information_type == CHILD_INFO_TYPE_MODULE_COW_SIZE) { + g_pserver->stat_module_cow_bytes = cow; + } +} + +/* Read child info data from the pipe. + * if complete data read into the buffer, + * data is stored into *buffer, and returns 1. + * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */ +int readChildInfo(childInfoType *information_type, size_t *cow, monotime *cow_updated, size_t *keys, double* progress) { + /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */ + static child_info_data buffer; + ssize_t wlen = sizeof(buffer); + + /* Do not overlap */ + if (g_pserver->child_info_nread == wlen) g_pserver->child_info_nread = 0; + + int nread = read(g_pserver->child_info_pipe[0], (char *)&buffer + g_pserver->child_info_nread, wlen - g_pserver->child_info_nread); + if (nread > 0) { + g_pserver->child_info_nread += nread; + } + + /* We have complete child info */ + if (g_pserver->child_info_nread == wlen) { + *information_type = buffer.information_type; + *cow = buffer.cow; + *cow_updated = buffer.cow_updated; + *keys = buffer.keys; + *progress = buffer.progress; + return 1; + } else { + return 0; + } +} + +/* Receive info data from child. */ +void receiveChildInfo(void) { + if (g_pserver->child_info_pipe[0] == -1) return; + + size_t cow; + monotime cow_updated; + size_t keys; + double progress; + childInfoType information_type; + + /* Drain the pipe and update child info so that we get the final message. */ + while (readChildInfo(&information_type, &cow, &cow_updated, &keys, &progress)) { + updateChildInfo(information_type, cow, cow_updated, keys, progress); } } diff --git a/src/cli_common.c b/src/cli_common.c new file mode 100644 index 000000000..e88327ace --- /dev/null +++ b/src/cli_common.c @@ -0,0 +1,194 @@ +/* CLI (command line interface) common methods + * + * Copyright (c) 2020, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "cli_common.h" +#include +#include +#include /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */ +#include /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */ +#ifdef USE_OPENSSL +#include +#include +#include +#endif + + +/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if + * not building with TLS support. + */ +int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err) { +#ifdef USE_OPENSSL + static SSL_CTX *ssl_ctx = NULL; + + if (!ssl_ctx) { + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + *err = "Failed to create SSL_CTX"; + goto error; + } + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, config.skip_cert_verify ? SSL_VERIFY_NONE : SSL_VERIFY_PEER, NULL); + + if (config.cacert || config.cacertdir) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, config.cacert, config.cacertdir)) { + *err = "Invalid CA Certificate File/Directory"; + goto error; + } + } else { + if (!SSL_CTX_set_default_verify_paths(ssl_ctx)) { + *err = "Failed to use default CA paths"; + goto error; + } + } + + if (config.cert && !SSL_CTX_use_certificate_chain_file(ssl_ctx, config.cert)) { + *err = "Invalid client certificate"; + goto error; + } + + if (config.key && !SSL_CTX_use_PrivateKey_file(ssl_ctx, config.key, SSL_FILETYPE_PEM)) { + *err = "Invalid private key"; + goto error; + } + if (config.ciphers && !SSL_CTX_set_cipher_list(ssl_ctx, config.ciphers)) { + *err = "Error while configuring ciphers"; + goto error; + } +#ifdef TLS1_3_VERSION + if (config.ciphersuites && !SSL_CTX_set_ciphersuites(ssl_ctx, config.ciphersuites)) { + *err = "Error while setting cypher suites"; + goto error; + } +#endif + } + + SSL *ssl = SSL_new(ssl_ctx); + if (!ssl) { + *err = "Failed to create SSL object"; + return REDIS_ERR; + } + + if (config.sni && !SSL_set_tlsext_host_name(ssl, config.sni)) { + *err = "Failed to configure SNI"; + SSL_free(ssl); + return REDIS_ERR; + } + + return redisInitiateSSL(c, ssl); + +error: + SSL_CTX_free(ssl_ctx); + ssl_ctx = NULL; + return REDIS_ERR; +#else + (void) config; + (void) c; + (void) err; + return REDIS_OK; +#endif +} + +/* Wrapper around hiredis to allow arbitrary reads and writes. + * + * We piggybacks on top of hiredis to achieve transparent TLS support, + * and use its internal buffers so it can co-exist with commands + * previously/later issued on the connection. + * + * Interface is close to enough to read()/write() so things should mostly + * work transparently. + */ + +/* Write a raw buffer through a redisContext. If we already have something + * in the buffer (leftovers from hiredis operations) it will be written + * as well. + */ +ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len) +{ + int done = 0; + + /* Append data to buffer which is *usually* expected to be empty + * but we don't assume that, and write. + */ + c->obuf = sdscatlen(c->obuf, buf, buf_len); + if (redisBufferWrite(c, &done) == REDIS_ERR) { + if (!(c->flags & REDIS_BLOCK)) + errno = EAGAIN; + + /* On error, we assume nothing was written and we roll back the + * buffer to its original state. + */ + if (sdslen(c->obuf) > buf_len) + sdsrange(c->obuf, 0, -(buf_len+1)); + else + sdsclear(c->obuf); + + return -1; + } + + /* If we're done, free up everything. We may have written more than + * buf_len (if c->obuf was not initially empty) but we don't have to + * tell. + */ + if (done) { + sdsclear(c->obuf); + return buf_len; + } + + /* Write was successful but we have some leftovers which we should + * remove from the buffer. + * + * Do we still have data that was there prior to our buf? If so, + * restore buffer to it's original state and report no new data was + * writen. + */ + if (sdslen(c->obuf) > buf_len) { + sdsrange(c->obuf, 0, -(buf_len+1)); + return 0; + } + + /* At this point we're sure no prior data is left. We flush the buffer + * and report how much we've written. + */ + size_t left = sdslen(c->obuf); + sdsclear(c->obuf); + return buf_len - left; +} + +/* Wrapper around OpenSSL (libssl and libcrypto) initialisation + */ +int cliSecureInit() +{ +#ifdef USE_OPENSSL + ERR_load_crypto_strings(); + SSL_load_error_strings(); + SSL_library_init(); +#endif + return REDIS_OK; +} diff --git a/src/cli_common.h b/src/cli_common.h new file mode 100644 index 000000000..77f28d805 --- /dev/null +++ b/src/cli_common.h @@ -0,0 +1,58 @@ +#ifndef __CLICOMMON_H +#define __CLICOMMON_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct cliSSLconfig { + /* Requested SNI, or NULL */ + char *sni; + /* CA Certificate file, or NULL */ + char *cacert; + /* Directory where trusted CA certificates are stored, or NULL */ + char *cacertdir; + /* Skip server certificate verification. */ + int skip_cert_verify; + /* Client certificate to authenticate with, or NULL */ + char *cert; + /* Private key file to authenticate with, or NULL */ + char *key; + /* Prefered cipher list, or NULL (applies only to <= TLSv1.2) */ + char* ciphers; + /* Prefered ciphersuites list, or NULL (applies only to TLSv1.3) */ + char* ciphersuites; +} cliSSLconfig; + +/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if + * not building with TLS support. + */ +int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err); + +/* Wrapper around hiredis to allow arbitrary reads and writes. + * + * We piggybacks on top of hiredis to achieve transparent TLS support, + * and use its internal buffers so it can co-exist with commands + * previously/later issued on the connection. + * + * Interface is close to enough to read()/write() so things should mostly + * work transparently. + */ + +/* Write a raw buffer through a redisContext. If we already have something + * in the buffer (leftovers from hiredis operations) it will be written + * as well. + */ +ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len); + +/* Wrapper around OpenSSL (libssl and libcrypto) initialisation. + */ +int cliSecureInit(); + +#ifdef __cplusplus +} +#endif + +#endif /* __CLICOMMON_H */ diff --git a/src/cluster.cpp b/src/cluster.cpp index 35c07b043..522670945 100644 --- a/src/cluster.cpp +++ b/src/cluster.cpp @@ -47,7 +47,7 @@ clusterNode *myself = NULL; clusterNode *createClusterNode(char *nodename, int flags); -int clusterAddNode(clusterNode *node); +void clusterAddNode(clusterNode *node); void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); void clusterReadHandler(connection *conn); void clusterSendPing(clusterLink *link, int type); @@ -55,7 +55,7 @@ void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); void clusterUpdateState(void); int clusterNodeGetSlotBit(clusterNode *n, int slot); -sds clusterGenNodesDescription(int filter); +sds clusterGenNodesDescription(int filter, int use_pport); clusterNode *clusterLookupNode(const char *name); int clusterNodeAddSlave(clusterNode *master, clusterNode *slave); int clusterAddSlot(clusterNode *n, int slot); @@ -198,6 +198,9 @@ int clusterLoadConfig(char *filename) { * base port. */ n->cport = busp ? atoi(busp) : n->port + CLUSTER_PORT_INCR; + /* The plaintext port for client in a TLS cluster (n->pport) is not + * stored in nodes.conf. It is received later over the bus protocol. */ + /* Parse flags */ p = s = argv[2]; while(p) { @@ -225,7 +228,7 @@ int clusterLoadConfig(char *filename) { } else if (!strcasecmp(s,"noflags")) { /* nothing to do */ } else { - serverPanic("Unknown flag in redis cluster config file"); + serverPanic("Unknown flag in KeyDB cluster config file"); } if (p) s = p+1; } @@ -353,7 +356,7 @@ int clusterSaveConfig(int do_fsync) { /* Get the nodes description and concatenate our "vars" directive to * save currentEpoch and lastVoteEpoch. */ - ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE); + ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE, 0); ci = sdscatprintf(ci,"vars currentEpoch %llu lastVoteEpoch %llu\n", (unsigned long long) g_pserver->cluster->currentEpoch, (unsigned long long) g_pserver->cluster->lastVoteEpoch); @@ -372,7 +375,7 @@ int clusterSaveConfig(int do_fsync) { if (write(fd,ci,sdslen(ci)) != (ssize_t)sdslen(ci)) goto err; if (do_fsync) { g_pserver->cluster->todo_before_sleep &= ~CLUSTER_TODO_FSYNC_CONFIG; - fsync(fd); + if (fsync(fd) == -1) goto err; } /* Truncate the file if needed to remove the final \n padding that @@ -415,7 +418,7 @@ int clusterLockConfig(char *filename) { /* To lock it, we need to open the file in a way it is created if * it does not exist, otherwise there is a race condition with other * processes. */ - int fd = open(filename,O_WRONLY|O_CREAT,0644); + int fd = open(filename,O_WRONLY|O_CREAT|O_CLOEXEC,0644); if (fd == -1) { serverLog(LL_WARNING, "Can't open %s in order to acquire a lock: %s", @@ -427,7 +430,7 @@ int clusterLockConfig(char *filename) { if (errno == EWOULDBLOCK) { serverLog(LL_WARNING, "Sorry, the cluster configuration file %s is already used " - "by a different Redis Cluster node. Please make sure that " + "by a different KeyDB Cluster node. Please make sure that " "different nodes use different cluster configuration " "files.", filename); } else { @@ -452,6 +455,26 @@ int clusterLockConfig(char *filename) { return C_OK; } +/* Derives our ports to be announced in the cluster bus. */ +void deriveAnnouncedPorts(int *announced_port, int *announced_pport, + int *announced_cport) { + int port = g_pserver->tls_cluster ? g_pserver->tls_port : g_pserver->port; + /* Default announced ports. */ + *announced_port = port; + *announced_pport = g_pserver->tls_cluster ? g_pserver->port : 0; + *announced_cport = port + CLUSTER_PORT_INCR; + /* Config overriding announced ports. */ + if (g_pserver->tls_cluster && g_pserver->cluster_announce_tls_port) { + *announced_port = g_pserver->cluster_announce_tls_port; + *announced_pport = g_pserver->cluster_announce_port; + } else if (g_pserver->cluster_announce_port) { + *announced_port = g_pserver->cluster_announce_port; + } + if (g_pserver->cluster_announce_bus_port) { + *announced_cport = g_pserver->cluster_announce_bus_port; + } +} + /* Some flags (currently just the NOFAILOVER flag) may need to be updated * in the "myself" node based on the current configuration of the node, * that may change at runtime via CONFIG SET. This function changes the @@ -519,33 +542,26 @@ void clusterInit(void) { if (saveconf) clusterSaveConfigOrDie(1); /* We need a listening TCP port for our cluster messaging needs. */ - g_pserver->cfd_count = 0; + g_pserver->cfd.count = 0; /* Port sanity check II * The other handshake port check is triggered too late to stop * us from trying to use a too-high cluster port number. */ int port = g_pserver->tls_cluster ? g_pserver->tls_port : g_pserver->port; if (port > (65535-CLUSTER_PORT_INCR)) { - serverLog(LL_WARNING, "Redis port number too high. " + serverLog(LL_WARNING, "KeyDB port number too high. " "Cluster communication port is 10,000 port " - "numbers higher than your Redis port. " - "Your Redis port number must be " - "lower than 55535."); + "numbers higher than your KeyDB port. " + "Your KeyDB port number must be 55535 or less."); exit(1); } - if (listenToPort(port+CLUSTER_PORT_INCR, - g_pserver->cfd,&g_pserver->cfd_count, 0 /*fReusePort*/, 0 /*fFirstListen*/) == C_ERR) - { + if (listenToPort(port+CLUSTER_PORT_INCR, &g_pserver->cfd, 0 /*fReusePort*/, 0 /*fFirstListen*/) == C_ERR) { exit(1); - } else { - int j; - - for (j = 0; j < g_pserver->cfd_count; j++) { - if (aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, g_pserver->cfd[j], AE_READABLE, - clusterAcceptHandler, NULL) == AE_ERR) - serverPanic("Unrecoverable error creating Redis Cluster " - "file event."); - } + } + + serverAssert(serverTL == &g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN]); + if (createSocketAcceptHandler(&g_pserver->cfd, clusterAcceptHandler) != C_OK) { + serverPanic("Unrecoverable error creating Redis Cluster socket accept handler."); } /* The slots -> keys map is a radix tree. Initialize it here. */ @@ -553,14 +569,9 @@ void clusterInit(void) { memset(g_pserver->cluster->slots_keys_count,0, sizeof(g_pserver->cluster->slots_keys_count)); - /* Set myself->port / cport to my listening ports, we'll just need to + /* Set myself->port/cport/pport to my listening ports, we'll just need to * discover the IP address via MEET messages. */ - myself->port = port; - myself->cport = port+CLUSTER_PORT_INCR; - if (g_pserver->cluster_announce_port) - myself->port = g_pserver->cluster_announce_port; - if (g_pserver->cluster_announce_bus_port) - myself->cport = g_pserver->cluster_announce_bus_port; + deriveAnnouncedPorts(&myself->port, &myself->pport, &myself->cport); g_pserver->cluster->mf_end = 0; resetManualFailover(); @@ -815,6 +826,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { node->configEpoch = 0; node->flags = flags; memset(node->slots,0,sizeof(node->slots)); + node->slots_info = NULL; node->numslots = 0; node->numslaves = 0; node->slaves = NULL; @@ -826,6 +838,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { memset(node->ip,0,sizeof(node->ip)); node->port = 0; node->cport = 0; + node->pport = 0; node->fail_reports = listCreate(); node->voted_time = 0; node->orphaned_time = 0; @@ -997,12 +1010,12 @@ void freeClusterNode(clusterNode *n) { } /* Add a node to the nodes hash table */ -int clusterAddNode(clusterNode *node) { +void clusterAddNode(clusterNode *node) { int retval; retval = dictAdd(g_pserver->cluster->nodes, sdsnewlen(node->name,CLUSTER_NAMELEN), node); - return (retval == DICT_OK) ? C_OK : C_ERR; + serverAssert(retval == DICT_OK); } /* Remove a node from the cluster. The function performs the high level @@ -1532,6 +1545,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { if (node->link) freeClusterLink(node->link); memcpy(node->ip,g->ip,NET_IP_STR_LEN); node->port = ntohs(g->port); + node->pport = ntohs(g->pport); node->cport = ntohs(g->cport); node->flags &= ~CLUSTER_NODE_NOADDR; } @@ -1553,6 +1567,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { node = createClusterNode(g->nodename, flags); memcpy(node->ip,g->ip,NET_IP_STR_LEN); node->port = ntohs(g->port); + node->pport = ntohs(g->pport); node->cport = ntohs(g->cport); clusterAddNode(node); } @@ -1592,6 +1607,7 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, { char ip[NET_IP_STR_LEN] = {0}; int port = ntohs(hdr->port); + int pport = ntohs(hdr->pport); int cport = ntohs(hdr->cport); /* We don't proceed if the link is the same as the sender link, as this @@ -1603,12 +1619,13 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, if (link == node->link) return 0; nodeIp2String(ip,link,hdr->myip); - if (node->port == port && node->cport == cport && + if (node->port == port && node->cport == cport && node->pport == pport && strcmp(ip,node->ip) == 0) return 0; /* IP / port is different, update it. */ memcpy(node->ip,ip,sizeof(ip)); node->port = port; + node->pport = pport; node->cport = cport; if (node->link) freeClusterLink(node->link); node->flags &= ~CLUSTER_NODE_NOADDR; @@ -1658,7 +1675,7 @@ void clusterSetNodeAsMaster(clusterNode *n) { * case we receive the info via an UPDATE packet. */ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoch, unsigned char *slots) { int j; - clusterNode *curmaster, *newmaster = NULL; + clusterNode *curmaster = NULL, *newmaster = NULL; /* The dirty slots list is a list of slots for which we lose the ownership * while having still keys inside. This usually happens after a failover * or after a manual cluster reconfiguration operated by the admin. @@ -1669,6 +1686,12 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc uint16_t dirty_slots[CLUSTER_SLOTS]; int dirty_slots_count = 0; + /* We should detect if sender is new master of our shard. + * We will know it if all our slots were migrated to sender, and sender + * has no slots except ours */ + int sender_slots = 0; + int migrated_our_slots = 0; + /* Here we set curmaster to this node or the node this node * replicates to if it's a slave. In the for loop we are * interested to check if slots are taken away from curmaster. */ @@ -1681,6 +1704,8 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc for (j = 0; j < CLUSTER_SLOTS; j++) { if (bitmapTestBit(slots,j)) { + sender_slots++; + /* The slot is already bound to the sender of this message. */ if (g_pserver->cluster->slots[j] == sender) continue; @@ -1707,8 +1732,10 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc dirty_slots_count++; } - if (g_pserver->cluster->slots[j] == curmaster) + if (g_pserver->cluster->slots[j] == curmaster) { newmaster = sender; + migrated_our_slots++; + } clusterDelSlot(j); clusterAddSlot(sender,j); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| @@ -1731,7 +1758,9 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc * master. * 2) We are a slave and our master is left without slots. We need * to replicate to the new slots owner. */ - if (newmaster && curmaster->numslots == 0) { + if (newmaster && curmaster->numslots == 0 && + (g_pserver->cluster_allow_replica_migration || + sender_slots == migrated_our_slots)) { serverLog(LL_WARNING, "Configuration change detected. Reconfiguring myself " "as a replica of %.40s", sender->name); @@ -1859,9 +1888,10 @@ int clusterProcessPacket(clusterLink *link) { nodeIsSlave(myself) && myself->slaveof == sender && hdr->mflags[0] & CLUSTERMSG_FLAG0_PAUSED && - g_pserver->cluster->mf_master_offset == 0) + g_pserver->cluster->mf_master_offset == -1) { g_pserver->cluster->mf_master_offset = sender->repl_offset; + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); serverLog(LL_WARNING, "Received replication offset for paused " "master manual failover: %lld", @@ -1909,6 +1939,7 @@ int clusterProcessPacket(clusterLink *link) { node = createClusterNode(NULL,CLUSTER_NODE_HANDSHAKE); nodeIp2String(node->ip,link,hdr->myip); node->port = ntohs(hdr->port); + node->pport = ntohs(hdr->pport); node->cport = ntohs(hdr->cport); clusterAddNode(node); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -1971,6 +2002,7 @@ int clusterProcessPacket(clusterLink *link) { link->node->flags |= CLUSTER_NODE_NOADDR; link->node->ip[0] = '\0'; link->node->port = 0; + link->node->pport = 0; link->node->cport = 0; freeClusterLink(link); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -2165,7 +2197,7 @@ int clusterProcessPacket(clusterLink *link) { /* Don't bother creating useless objects if there are no * Pub/Sub subscribers. */ if (dictSize(g_pserver->pubsub_channels) || - listLength(g_pserver->pubsub_patterns)) + dictSize(g_pserver->pubsub_patterns)) { channel_len = ntohl(hdr->data.publish.msg.channel_len); message_len = ntohl(hdr->data.publish.msg.message_len); @@ -2203,9 +2235,15 @@ int clusterProcessPacket(clusterLink *link) { resetManualFailover(); g_pserver->cluster->mf_end = now + CLUSTER_MF_TIMEOUT; g_pserver->cluster->mf_slave = sender; - pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT)); + pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT),CLIENT_PAUSE_WRITE); serverLog(LL_WARNING,"Manual failover requested by replica %.40s.", sender->name); + /* We need to send a ping message to the replica, as it would carry + * `server.cluster->mf_master_offset`, which means the master paused clients + * at offset `server.cluster->mf_master_offset`, so that the replica would + * know that it is safe to set its `server.cluster->mf_can_start` to 1 so as + * to complete failover as quickly as possible. */ + clusterSendPing(link, CLUSTERMSG_TYPE_PING); } else if (type == CLUSTERMSG_TYPE_UPDATE) { clusterNode *n; /* The node the update is about. */ uint64_t reportedConfigEpoch = @@ -2472,19 +2510,16 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) { hdr->myip[NET_IP_STR_LEN-1] = '\0'; } - /* Handle cluster-announce-port as well. */ - int port = g_pserver->tls_cluster ? g_pserver->tls_port : g_pserver->port; - int announced_port = g_pserver->cluster_announce_port ? - g_pserver->cluster_announce_port : port; - int announced_cport = g_pserver->cluster_announce_bus_port ? - g_pserver->cluster_announce_bus_port : - (port + CLUSTER_PORT_INCR); + /* Handle cluster-announce-[tls-|bus-]port. */ + int announced_port, announced_pport, announced_cport; + deriveAnnouncedPorts(&announced_port, &announced_pport, &announced_cport); memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots)); memset(hdr->slaveof,0,CLUSTER_NAMELEN); if (myself->slaveof != NULL) memcpy(hdr->slaveof,myself->slaveof->name, CLUSTER_NAMELEN); hdr->port = htons(announced_port); + hdr->pport = htons(announced_pport); hdr->cport = htons(announced_cport); hdr->flags = htons(myself->flags); hdr->state = g_pserver->cluster->state; @@ -2541,6 +2576,7 @@ void clusterSetGossipEntry(clusterMsg *hdr, int i, clusterNode *n) { gossip->port = htons(n->port); gossip->cport = htons(n->cport); gossip->flags = htons(n->flags); + gossip->pport = htons(n->pport); gossip->notused1 = 0; } @@ -2856,7 +2892,7 @@ void clusterPropagatePublish(robj *channel, robj *message) { * SLAVE node specific functions * -------------------------------------------------------------------------- */ -/* This function sends a FAILOVE_AUTH_REQUEST message to every node in order to +/* This function sends a FAILOVER_AUTH_REQUEST message to every node in order to * see if there is the quorum for this slave instance to failover its failing * master. * @@ -3139,7 +3175,7 @@ void clusterFailoverReplaceYourMaster(void) { /* This function is called if we are a slave node and our master serving * a non-zero amount of hash slots is in FAIL state. * - * The gaol of this function is: + * The goal of this function is: * 1) To check if we are able to perform a failover, is our data updated? * 2) Try to get elected by masters. * 3) Perform the failover informing all the other nodes. @@ -3184,7 +3220,7 @@ void clusterHandleSlaveFailover(void) { return; } - /* Set data_age to the number of seconds we are disconnected from + /* Set data_age to the number of milliseconds we are disconnected from * the master. */ if (getFirstMaster()->repl_state == REPL_STATE_CONNECTED) { data_age = (mstime_t)(g_pserver->unixtime - getFirstMaster()->master->lastinteraction) @@ -3462,14 +3498,13 @@ void clusterHandleSlaveMigration(int max_slaves) { * The function can be used both to initialize the manual failover state at * startup or to abort a manual failover in progress. */ void resetManualFailover(void) { - if (g_pserver->cluster->mf_end && clientsArePaused()) { - g_pserver->clients_pause_end_time = 0; - unpauseClientsIfNecessary(); + if (g_pserver->cluster->mf_end) { + checkClientPauseTimeoutAndReturnIfPaused(); } g_pserver->cluster->mf_end = 0; /* No manual failover in progress. */ g_pserver->cluster->mf_can_start = 0; g_pserver->cluster->mf_slave = NULL; - g_pserver->cluster->mf_master_offset = 0; + g_pserver->cluster->mf_master_offset = -1; } /* If a manual failover timed out, abort it. */ @@ -3490,7 +3525,7 @@ void clusterHandleManualFailover(void) { * next steps are performed by clusterHandleSlaveFailover(). */ if (g_pserver->cluster->mf_can_start) return; - if (g_pserver->cluster->mf_master_offset == 0) return; /* Wait for offset... */ + if (g_pserver->cluster->mf_master_offset == -1) return; /* Wait for offset... */ if (g_pserver->cluster->mf_master_offset == replicationGetSlaveOffset(getFirstMaster())) { /* Our replication offset matches the master replication offset @@ -3499,7 +3534,10 @@ void clusterHandleManualFailover(void) { serverLog(LL_WARNING, "All master replication stream processed, " "manual failover can start."); + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_FAILOVER); + return; } + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); } /* ----------------------------------------------------------------------------- @@ -3677,7 +3715,6 @@ void clusterCron(void) { now - node->link->ctime > g_pserver->cluster_node_timeout && /* was not already reconnected */ node->ping_sent && /* we already sent a ping */ - node->pong_received < node->ping_sent && /* still waiting pong */ /* and we are waiting for the pong more than timeout/2 */ ping_delay > g_pserver->cluster_node_timeout/2 && /* and in such interval we are not seeing any traffic at all. */ @@ -3760,7 +3797,8 @@ void clusterCron(void) { * the orphaned masters. Note that it does not make sense to try * a migration if there is no master with at least *two* working * slaves. */ - if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves) + if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves && + g_pserver->cluster_allow_replica_migration) clusterHandleSlaveMigration(max_slaves); } @@ -3774,25 +3812,35 @@ void clusterCron(void) { * handlers, or to perform potentially expansive tasks that we need to do * a single time before replying to clients. */ void clusterBeforeSleep(void) { - /* Handle failover, this is needed when it is likely that there is already - * the quorum from masters in order to react fast. */ - if (g_pserver->cluster->todo_before_sleep & CLUSTER_TODO_HANDLE_FAILOVER) - clusterHandleSlaveFailover(); - - /* Update the cluster state. */ - if (g_pserver->cluster->todo_before_sleep & CLUSTER_TODO_UPDATE_STATE) - clusterUpdateState(); - - /* Save the config, possibly using fsync. */ - if (g_pserver->cluster->todo_before_sleep & CLUSTER_TODO_SAVE_CONFIG) { - int fsync = g_pserver->cluster->todo_before_sleep & - CLUSTER_TODO_FSYNC_CONFIG; - clusterSaveConfigOrDie(fsync); - } + int flags = g_pserver->cluster->todo_before_sleep; /* Reset our flags (not strictly needed since every single function * called for flags set should be able to clear its flag). */ g_pserver->cluster->todo_before_sleep = 0; + + if (flags & CLUSTER_TODO_HANDLE_MANUALFAILOVER) { + /* Handle manual failover as soon as possible so that won't have a 100ms + * as it was handled only in clusterCron */ + if(nodeIsSlave(myself)) { + clusterHandleManualFailover(); + if (!(g_pserver->cluster_module_flags & CLUSTER_MODULE_FLAG_NO_FAILOVER)) + clusterHandleSlaveFailover(); + } + } else if (flags & CLUSTER_TODO_HANDLE_FAILOVER) { + /* Handle failover, this is needed when it is likely that there is already + * the quorum from masters in order to react fast. */ + clusterHandleSlaveFailover(); + } + + /* Update the cluster state. */ + if (flags & CLUSTER_TODO_UPDATE_STATE) + clusterUpdateState(); + + /* Save the config, possibly using fsync. */ + if (flags & CLUSTER_TODO_SAVE_CONFIG) { + int fsync = flags & CLUSTER_TODO_FSYNC_CONFIG; + clusterSaveConfigOrDie(fsync); + } } void clusterDoBeforeSleep(int flags) { @@ -3860,7 +3908,7 @@ int clusterNodeSetSlotBit(clusterNode *n, int slot) { * However new masters with slots assigned are considered valid * migration targets if the rest of the cluster is not a slave-less. * - * See https://github.com/antirez/redis/issues/3043 for more info. */ + * See https://github.com/redis/redis/issues/3043 for more info. */ if (n->numslots == 1 && clusterMastersHaveSlaves()) n->flags |= CLUSTER_NODE_MIGRATE_TO; } @@ -4168,55 +4216,62 @@ sds representClusterNodeFlags(sds ci, uint16_t flags) { * See clusterGenNodesDescription() top comment for more information. * * The function returns the string representation as an SDS string. */ -sds clusterGenNodeDescription(clusterNode *node) { +sds clusterGenNodeDescription(clusterNode *node, int use_pport) { int j, start; sds ci; + int port = use_pport && node->pport ? node->pport : node->port; /* Node coordinates */ - ci = sdscatprintf(sdsempty(),"%.40s %s:%d@%d ", - node->name, + ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); + ci = sdscatfmt(ci," %s:%i@%i ", node->ip, - node->port, + port, node->cport); /* Flags */ ci = representClusterNodeFlags(ci, node->flags); /* Slave of... or just "-" */ + ci = sdscatlen(ci," ",1); if (node->slaveof) - ci = sdscatprintf(ci," %.40s ",node->slaveof->name); + ci = sdscatlen(ci,node->slaveof->name,CLUSTER_NAMELEN); else - ci = sdscatlen(ci," - ",3); + ci = sdscatlen(ci,"-",1); unsigned long long nodeEpoch = node->configEpoch; if (nodeIsSlave(node) && node->slaveof) { nodeEpoch = node->slaveof->configEpoch; } /* Latency from the POV of this node, config epoch, link status */ - ci = sdscatprintf(ci,"%lld %lld %llu %s", + ci = sdscatfmt(ci," %I %I %U %s", (long long) node->ping_sent, (long long) node->pong_received, nodeEpoch, (node->link || node->flags & CLUSTER_NODE_MYSELF) ? "connected" : "disconnected"); - /* Slots served by this instance */ - start = -1; - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit; + /* Slots served by this instance. If we already have slots info, + * append it diretly, otherwise, generate slots only if it has. */ + if (node->slots_info) { + ci = sdscatsds(ci, node->slots_info); + } else if (node->numslots > 0) { + start = -1; + for (j = 0; j < CLUSTER_SLOTS; j++) { + int bit; - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - if (bit && j == CLUSTER_SLOTS-1) j++; - - if (start == j-1) { - ci = sdscatprintf(ci," %d",start); - } else { - ci = sdscatprintf(ci," %d-%d",start,j-1); + if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { + if (start == -1) start = j; + } + if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { + if (bit && j == CLUSTER_SLOTS-1) j++; + + if (start == j-1) { + ci = sdscatfmt(ci," %i",start); + } else { + ci = sdscatfmt(ci," %i-%i",start,j-1); + } + start = -1; } - start = -1; } } @@ -4237,6 +4292,41 @@ sds clusterGenNodeDescription(clusterNode *node) { return ci; } +/* Generate the slot topology for all nodes and store the string representation + * in the slots_info struct on the node. This is used to improve the efficiency + * of clusterGenNodesDescription() because it removes looping of the slot space + * for generating the slot info for each node individually. */ +void clusterGenNodesSlotsInfo(int filter) { + clusterNode *n = NULL; + int start = -1; + + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; + continue; + } + + /* Generate slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != g_pserver->cluster->slots[i]) { + if (!(n->flags & filter)) { + if (n->slots_info == NULL) n->slots_info = sdsempty(); + if (start == i-1) { + n->slots_info = sdscatfmt(n->slots_info," %i",start); + } else { + n->slots_info = sdscatfmt(n->slots_info," %i-%i",start,i-1); + } + } + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; + } + } +} + /* Generate a csv-alike representation of the nodes we are aware of, * including the "myself" node, and return an SDS string containing the * representation (it is up to the caller to free it). @@ -4246,23 +4336,35 @@ sds clusterGenNodeDescription(clusterNode *node) { * include all the known nodes in the representation, including nodes in * the HANDSHAKE state. * + * Setting use_pport to 1 in a TLS cluster makes the result contain the + * plaintext client port rather then the TLS client port of each node. + * * The representation obtained using this function is used for the output * of the CLUSTER NODES function, and as format for the cluster * configuration file (nodes.conf) for a given node. */ -sds clusterGenNodesDescription(int filter) { +sds clusterGenNodesDescription(int filter, int use_pport) { sds ci = sdsempty(), ni; dictIterator *di; dictEntry *de; + /* Generate all nodes slots info firstly. */ + clusterGenNodesSlotsInfo(filter); + di = dictGetSafeIterator(g_pserver->cluster->nodes); while((de = dictNext(di)) != NULL) { clusterNode *node = (clusterNode*)dictGetVal(de); if (node->flags & filter) continue; - ni = clusterGenNodeDescription(node); + ni = clusterGenNodeDescription(node, use_pport); ci = sdscatsds(ci,ni); sdsfree(ni); ci = sdscatlen(ci,"\n",1); + + /* Release slots info. */ + if (node->slots_info) { + sdsfree(node->slots_info); + node->slots_info = NULL; + } } dictReleaseIterator(di); return ci; @@ -4300,7 +4402,37 @@ int getSlotOrReply(client *c, robj *o) { return (int) slot; } -void clusterReplyMultiBulkSlots(client *c) { +void addNodeReplyForClusterSlot(client *c, clusterNode *node, int start_slot, int end_slot) { + int i, nested_elements = 3; /* slots (2) + master addr (1) */ + void *nested_replylen = addReplyDeferredLen(c); + addReplyLongLong(c, start_slot); + addReplyLongLong(c, end_slot); + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->ip); + /* Report non-TLS ports to non-TLS client in TLS cluster if available. */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + addReplyLongLong(c, use_pport && node->pport ? node->pport : node->port); + addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); + + /* Remaining nodes in reply are replicas for slot range */ + for (i = 0; i < node->numslaves; i++) { + /* This loop is copy/pasted from clusterGenNodeDescription() + * with modifications for per-slot node aggregation. */ + if (nodeFailed(node->slaves[i])) continue; + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->slaves[i]->ip); + /* Report slave's non-TLS port to non-TLS client in TLS cluster */ + addReplyLongLong(c, (use_pport && node->slaves[i]->pport ? + node->slaves[i]->pport : + node->slaves[i]->port)); + addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); + nested_elements++; + } + setDeferredArrayLen(c, nested_replylen, nested_elements); +} + +void clusterReplyMultiBulkSlots(client * c) { /* Format: 1) 1) start slot * 2) end slot * 3) 1) master IP @@ -4311,69 +4443,29 @@ void clusterReplyMultiBulkSlots(client *c) { * 3) node ID * ... continued until done */ - - int num_masters = 0; + clusterNode *n = NULL; + int num_masters = 0, start = -1; void *slot_replylen = addReplyDeferredLen(c); - dictEntry *de; - dictIterator *di = dictGetSafeIterator(g_pserver->cluster->nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = (clusterNode*)dictGetVal(de); - int j = 0, start = -1; - int i, nested_elements = 0; - - /* Skip slaves (that are iterated when producing the output of their - * master) and masters not serving any slot. */ - if (!nodeIsMaster(node) || node->numslots == 0) continue; - - for(i = 0; i < node->numslaves; i++) { - if (nodeFailed(node->slaves[i])) continue; - nested_elements++; + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; + continue; } - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit, i; - - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - addReplyArrayLen(c, nested_elements + 3); /* slots (2) + master addr (1). */ - - if (bit && j == CLUSTER_SLOTS-1) j++; - - /* If slot exists in output map, add to it's list. - * else, create a new output map for this slot */ - if (start == j-1) { - addReplyLongLong(c, start); /* only one slot; low==high */ - addReplyLongLong(c, start); - } else { - addReplyLongLong(c, start); /* low */ - addReplyLongLong(c, j-1); /* high */ - } - start = -1; - - /* First node reply position is always the master */ - addReplyArrayLen(c, 3); - addReplyBulkCString(c, node->ip); - addReplyLongLong(c, node->port); - addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); - - /* Remaining nodes in reply are replicas for slot range */ - for (i = 0; i < node->numslaves; i++) { - /* This loop is copy/pasted from clusterGenNodeDescription() - * with modifications for per-slot node aggregation */ - if (nodeFailed(node->slaves[i])) continue; - addReplyArrayLen(c, 3); - addReplyBulkCString(c, node->slaves[i]->ip); - addReplyLongLong(c, node->slaves[i]->port); - addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); - } - num_masters++; - } + /* Add cluster slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != g_pserver->cluster->slots[i]) { + addNodeReplyForClusterSlot(c, n, start, i-1); + num_masters++; + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; } } - dictReleaseIterator(di); setDeferredArrayLen(c, slot_replylen, num_masters); } @@ -4385,28 +4477,49 @@ void clusterCommand(client *c) { if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"help")) { const char *help[] = { -"ADDSLOTS [slot ...] -- Assign slots to current node.", -"BUMPEPOCH -- Advance the cluster config epoch.", -"COUNT-failure-reports -- Return number of failure reports for .", -"COUNTKEYSINSLOT - Return the number of keys in .", -"DELSLOTS [slot ...] -- Delete slots information from current node.", -"FAILOVER [force|takeover] -- Promote current replica node to being a master.", -"FORGET -- Remove a node from the cluster.", -"GETKEYSINSLOT -- Return key names stored by current node in a slot.", -"FLUSHSLOTS -- Delete current node own slots information.", -"INFO - Return information about the cluster.", -"KEYSLOT -- Return the hash slot for .", -"MEET [bus-port] -- Connect nodes into a working cluster.", -"MYID -- Return the node id.", -"NODES -- Return cluster configuration seen by node. Output format:", -" ... ", -"REPLICATE -- Configure current node as replica to .", -"RESET [hard|soft] -- Reset current node (default: soft).", -"SET-config-epoch - Set config epoch of current node.", -"SETSLOT (importing|migrating|stable|node ) -- Set slot state.", -"REPLICAS -- Return replicas.", -"SAVECONFIG - Force saving cluster configuration on disk.", -"SLOTS -- Return information about slots range mappings. Each range is made of:", +"ADDSLOTS [ ...]", +" Assign slots to current node.", +"BUMPEPOCH", +" Advance the cluster config epoch.", +"COUNT-FAILURE-REPORTS ", +" Return number of failure reports for .", +"COUNTKEYSINSLOT ", +" Return the number of keys in .", +"DELSLOTS [ ...]", +" Delete slots information from current node.", +"FAILOVER [FORCE|TAKEOVER]", +" Promote current replica node to being a master.", +"FORGET ", +" Remove a node from the cluster.", +"GETKEYSINSLOT ", +" Return key names stored by current node in a slot.", +"FLUSHSLOTS", +" Delete current node own slots information.", +"INFO", +" Return information about the cluster.", +"KEYSLOT ", +" Return the hash slot for .", +"MEET []", +" Connect nodes into a working cluster.", +"MYID", +" Return the node id.", +"NODES", +" Return cluster configuration seen by node. Output format:", +" ...", +"REPLICATE ", +" Configure current node as replica to .", +"RESET [HARD|SOFT]", +" Reset current node (default: soft).", +"SET-CONFIG-EPOCH ", +" Set config epoch of current node.", +"SETSLOT (IMPORTING|MIGRATING|STABLE|NODE )", +" Set slot state.", +"REPLICAS ", +" Return replicas.", +"SAVECONFIG", +" Force saving cluster configuration on disk.", +"SLOTS", +" Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL }; @@ -4441,7 +4554,11 @@ NULL } } else if (!strcasecmp(szFromObj(c->argv[1]),"nodes") && c->argc == 2) { /* CLUSTER NODES */ - sds nodes = clusterGenNodesDescription(0); + /* Report plaintext ports, only if cluster is TLS but client is known to + * be non-TLS). */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + sds nodes = clusterGenNodesDescription(0, use_pport); addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); sdsfree(nodes); } else if (!strcasecmp(szFromObj(c->argv[1]),"myid") && c->argc == 2) { @@ -4577,6 +4694,9 @@ NULL g_pserver->cluster->migrating_slots_to[slot]) g_pserver->cluster->migrating_slots_to[slot] = NULL; + clusterDelSlot(slot); + clusterAddSlot(n,slot); + /* If this node was importing this slot, assigning the slot to * itself also clears the importing status. */ if (n == myself && @@ -4596,9 +4716,10 @@ NULL "configEpoch updated after importing slot %d", slot); } g_pserver->cluster->importing_slots_from[slot] = NULL; + /* After importing this slot, let the other nodes know as + * soon as possible. */ + clusterBroadcastPong(CLUSTER_BROADCAST_ALL); } - clusterDelSlot(slot); - clusterAddSlot(n,slot); } else { addReplyError(c, "Invalid CLUSTER SETSLOT action or number of arguments. Try CLUSTER HELP"); @@ -4813,9 +4934,12 @@ NULL return; } + /* Use plaintext port if cluster is TLS but client is non-TLS. */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); addReplyArrayLen(c,n->numslaves); for (j = 0; j < n->numslaves; j++) { - sds ni = clusterGenNodeDescription(n->slaves[j]); + sds ni = clusterGenNodeDescription(n->slaves[j], use_pport); addReplyBulkCString(c,ni); sdsfree(ni); } @@ -4844,7 +4968,7 @@ NULL takeover = 1; force = 1; /* Takeover also implies force. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -4935,7 +5059,7 @@ NULL } else if (!strcasecmp(szFromObj(c->argv[2]),"soft")) { hard = 0; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -5012,6 +5136,9 @@ int verifyDumpPayload(unsigned char *p, size_t len) { rdbver = (footer[1] << 8) | footer[0]; if (rdbver > RDB_VERSION) return C_ERR; + if (cserver.skip_checksum_validation) + return C_OK; + /* Verify CRC64 */ crc = crc64(0,p,len-8); memrev64ifbe(&crc); @@ -5071,7 +5198,7 @@ void mvccrestoreCommand(client *c) { setMvccTstamp(obj, mvcc); /* Create the key and set the TTL if any */ - if (dbMerge(c->db,key,obj,true)) { + if (dbMerge(c->db,szFromObj(key),obj,true)) { if (expire >= 0) { setExpire(c,c->db,key,nullptr,expire); } @@ -5120,7 +5247,7 @@ void restoreCommand(client *c) { } j++; /* Consume additional arg. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -5128,7 +5255,7 @@ void restoreCommand(client *c) { /* Make sure this key does not already exist here... */ robj *key = c->argv[1]; if (!replace && lookupKeyWrite(c->db,key) != NULL) { - addReply(c,shared.busykeyerr); + addReplyErrorObject(c,shared.busykeyerr); return; } @@ -5256,8 +5383,7 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti conn = g_pserver->tls_cluster ? connCreateTLS() : connCreateSocket(); if (connBlockingConnect(conn, szFromObj(c->argv[1]), atoi(szFromObj(c->argv[2])), timeout) != C_OK) { - addReplySds(c, - sdsnew("-IOERR error or timeout connecting to the client\r\n")); + addReplyError(c,"-IOERR error or timeout connecting to the client"); connClose(conn); sdsfree(name); return NULL; @@ -5344,7 +5470,7 @@ void migrateCommand(client *c) { replace = 1; } else if (!strcasecmp(szFromObj(c->argv[j]),"auth")) { if (!moreargs) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } j++; @@ -5367,7 +5493,7 @@ void migrateCommand(client *c) { num_keys = c->argc - j - 1; break; /* All the remaining args are keys. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -5861,18 +5987,14 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * cluster is down. */ if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; return NULL; - } else if (!(cmd->flags & CMD_READONLY) && !(cmd->proc == evalCommand) - && !(cmd->proc == evalShaCommand)) - { - /* The cluster is configured to allow read only commands - * but this command is neither readonly, nor EVAL or - * EVALSHA. */ + } else if (cmd->flags & CMD_WRITE) { + /* The cluster is configured to allow read only commands */ if (error_code) *error_code = CLUSTER_REDIR_DOWN_RO_STATE; return NULL; } else { /* Fall through and allow the command to be executed: - * this happens when g_pserver->cluster_allow_reads_when_down is - * true and the command is a readonly command or EVAL / EVALSHA. */ + * this happens when server.cluster_allow_reads_when_down is + * true and the command is not a write command */ } } @@ -5910,11 +6032,10 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in /* Handle the read-only client case reading from a slave: if this * node is a slave and the request is about a hash slot our master * is serving, we can reply without redirection. */ - int is_readonly_command = (c->cmd->flags & CMD_READONLY) || - (c->cmd->proc == execCommand && !(c->mstate.cmd_inv_flags & CMD_READONLY)); + int is_write_command = (c->cmd->flags & CMD_WRITE) || + (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE)); if (c->flags & CLIENT_READONLY && - (is_readonly_command || cmd->proc == evalCommand || - cmd->proc == evalShaCommand) && + !is_write_command && nodeIsSlave(myself) && myself->slaveof == n) { @@ -5936,25 +6057,30 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * be set to the hash slot that caused the redirection. */ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_code) { if (error_code == CLUSTER_REDIR_CROSS_SLOT) { - addReplySds(c,sdsnew("-CROSSSLOT Keys in request don't hash to the same slot\r\n")); + addReplyError(c,"-CROSSSLOT Keys in request don't hash to the same slot"); } else if (error_code == CLUSTER_REDIR_UNSTABLE) { /* The request spawns multiple keys in the same slot, * but the slot is not "stable" currently as there is * a migration or import in progress. */ - addReplySds(c,sdsnew("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")); + addReplyError(c,"-TRYAGAIN Multiple keys request during rehashing of slot"); } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { - addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down\r\n")); + addReplyError(c,"-CLUSTERDOWN The cluster is down"); } else if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { - addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down and only accepts read commands\r\n")); + addReplyError(c,"-CLUSTERDOWN The cluster is down and only accepts read commands"); } else if (error_code == CLUSTER_REDIR_DOWN_UNBOUND) { - addReplySds(c,sdsnew("-CLUSTERDOWN Hash slot not served\r\n")); + addReplyError(c,"-CLUSTERDOWN Hash slot not served"); } else if (error_code == CLUSTER_REDIR_MOVED || error_code == CLUSTER_REDIR_ASK) { - addReplySds(c,sdscatprintf(sdsempty(), - "-%s %d %s:%d\r\n", + /* Redirect to IP:port. Include plaintext port if cluster is TLS but + * client is non-TLS. */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + int port = use_pport && n->pport ? n->pport : n->port; + addReplyErrorSds(c,sdscatprintf(sdsempty(), + "-%s %d %s:%d", (error_code == CLUSTER_REDIR_ASK) ? "ASK" : "MOVED", - hashslot,n->ip,n->port)); + hashslot, n->ip, port)); } else { serverPanic("getNodeByQuery() unknown error."); } @@ -6006,6 +6132,15 @@ int clusterRedirectBlockedClientIfNeeded(client *c) { node = myself; } + /* if the client is read-only and attempting to access key that our + * replica can handle, allow it. */ + if ((c->flags & CLIENT_READONLY) && + !(c->lastcmd->flags & CMD_WRITE) && + nodeIsSlave(myself) && myself->slaveof == node) + { + node = myself; + } + /* We send an error and unblock the client if: * 1) The slot is unassigned, emitting a cluster down error. * 2) The slot is not handled by this node, nor being imported. */ diff --git a/src/cluster.h b/src/cluster.h index 6dfe318e2..db82c3f24 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -83,6 +83,7 @@ typedef struct clusterLink { #define CLUSTER_TODO_UPDATE_STATE (1<<1) #define CLUSTER_TODO_SAVE_CONFIG (1<<2) #define CLUSTER_TODO_FSYNC_CONFIG (1<<3) +#define CLUSTER_TODO_HANDLE_MANUALFAILOVER (1<<4) /* Message types. * @@ -121,6 +122,7 @@ typedef struct clusterNode { int flags; /* CLUSTER_NODE_... */ uint64_t configEpoch; /* Last configEpoch observed for this node */ unsigned char slots[CLUSTER_SLOTS/8]; /* slots handled by this node */ + sds slots_info; /* Slots info represented by string. */ int numslots; /* Number of slots handled by this node */ int numslaves; /* Number of slave nodes, if this is a master */ struct clusterNode **slaves; /* pointers to slave nodes */ @@ -137,7 +139,9 @@ typedef struct clusterNode { mstime_t orphaned_time; /* Starting time of orphaned master condition */ long long repl_offset; /* Last known repl offset for this node. */ char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ - int port; /* Latest known clients port of this node */ + int port; /* Latest known clients port (TLS or plain). */ + int pport; /* Latest known clients plaintext port. Only used + if the main clients port is for TLS. */ int cport; /* Latest known cluster port of this node. */ clusterLink *link; /* TCP/IP link with this node */ list *fail_reports; /* List of nodes signaling this as failing */ @@ -170,7 +174,7 @@ typedef struct clusterState { clusterNode *mf_slave; /* Slave performing the manual failover. */ /* Manual failover state of slave. */ long long mf_master_offset; /* Master offset the slave needs to start MF - or zero if still not received. */ + or -1 if still not received. */ int mf_can_start; /* If non-zero signal that the manual failover can start requesting masters vote. */ /* The following fields are used by masters to take state on elections. */ @@ -196,7 +200,8 @@ typedef struct { uint16_t port; /* base port last time it was seen */ uint16_t cport; /* cluster port last time it was seen */ uint16_t flags; /* node->flags copy */ - uint32_t notused1; + uint16_t pport; /* plaintext-port, when base port is TLS */ + uint16_t notused1; } clusterMsgDataGossip; typedef struct { @@ -269,7 +274,8 @@ typedef struct { unsigned char myslots[CLUSTER_SLOTS/8]; char slaveof[CLUSTER_NAMELEN]; char myip[NET_IP_STR_LEN]; /* Sender IP, if not all zeroed. */ - char notused1[34]; /* 34 bytes reserved for future usage. */ + char notused1[32]; /* 32 bytes reserved for future usage. */ + uint16_t pport; /* Sender TCP plaintext port, if base port is TLS */ uint16_t cport; /* Sender TCP cluster bus port */ uint16_t flags; /* Sender node flags */ unsigned char state; /* Cluster state from the POV of the sender */ diff --git a/src/compactvector.h b/src/compactvector.h index 222eaab2a..86b3d0efd 100644 --- a/src/compactvector.h +++ b/src/compactvector.h @@ -81,7 +81,7 @@ public: m_max = m_max + 4; m_data = (T*)zrealloc(m_data, sizeof(T) * m_max, MALLOC_LOCAL); - m_max = zmalloc_usable(m_data) / sizeof(T); + m_max = zmalloc_usable_size(m_data) / sizeof(T); } assert(idx < m_max); where = m_data + idx; diff --git a/src/config.cpp b/src/config.cpp index 9d7f14007..a10cdbe12 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -125,6 +125,19 @@ configEnum oom_score_adj_enum[] = { {NULL, 0} }; +configEnum acl_pubsub_default_enum[] = { + {"allchannels", USER_FLAG_ALLCHANNELS}, + {"resetchannels", 0}, + {NULL, 0} +}; + +configEnum sanitize_dump_payload_enum[] = { + {"no", SANITIZE_DUMP_NO}, + {"yes", SANITIZE_DUMP_YES}, + {"clients", SANITIZE_DUMP_CLIENTS}, + {NULL, 0} +}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -165,6 +178,15 @@ typedef struct stringConfigData { be stored as a NULL value. */ } stringConfigData; +typedef struct sdsConfigData { + sds *config; /* Pointer to the server config this value is stored in. */ + const char *default_value; /* Default value of the config on rewrite. */ + int (*is_valid_fn)(sds val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(sds val, sds prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int convert_empty_to_null; /* Boolean indicating if empty SDS strings should + be stored as a NULL value. */ +} sdsConfigData; + typedef struct enumConfigData { int *config; /* The pointer to the server config this value is stored in */ configEnum *enum_value; /* The underlying enum type this data represents */ @@ -211,6 +233,7 @@ typedef struct numericConfigData { typedef union typeData { boolConfigData yesno; stringConfigData string; + sdsConfigData sds; enumConfigData enumd; numericConfigData numeric; } typeData; @@ -218,8 +241,6 @@ typedef union typeData { typedef struct typeInterface { /* Called on server start, to init the server with default value */ void (*init)(typeData data); - /* Called on server start, should return 1 on success, 0 on error and should set err */ - int (*load)(typeData data, sds *argc, int argv, const char **err); /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error * and can set a verbose err string, update is true when called from CONFIG SET */ int (*set)(typeData data, sds value, int update, const char **err); @@ -232,11 +253,16 @@ typedef struct typeInterface { typedef struct standardConfig { const char *name; /* The user visible name of this config */ const char *alias; /* An alias that can also be used for this config */ - const int modifiable; /* Can this value be updated by CONFIG SET? */ + const unsigned int flags; /* Flags for this specific config */ typeInterface interface; /* The function pointers that define the type interface */ typeData data; /* The type specific data exposed used by the interface */ } standardConfig; +#define MODIFIABLE_CONFIG 0 /* This is the implied default for a standard + * config, which is mutable. */ +#define IMMUTABLE_CONFIG (1ULL<<0) /* Can this value only be set at startup? */ +#define SENSITIVE_CONFIG (1ULL<<1) /* Does this value contain sensitive information */ + extern standardConfig configs[]; /*----------------------------------------------------------------------------- @@ -440,6 +466,7 @@ void loadServerConfigFromString(char *config) { int linenum = 0, totlines, i; int slaveof_linenum = 0; sds *lines; + int save_loaded = 0; lines = sdssplitlen(config,strlen(config),"\n",1,&totlines); @@ -512,6 +539,14 @@ void loadServerConfigFromString(char *config) { err = "Invalid socket file permissions"; goto loaderr; } } else if (!strcasecmp(argv[0],"save")) { + /* We don't reset save params before loading, because if they're not part + * of the file the defaults should be used. + */ + if (!save_loaded) { + save_loaded = 1; + resetServerSaveParams(); + } + if (argc == 3) { int seconds = atoi(argv[1]); int changes = atoi(argv[2]); @@ -545,9 +580,7 @@ void loadServerConfigFromString(char *config) { fclose(logfp); } } else if (!strcasecmp(argv[0],"include") && argc == 2) { - loadServerConfig(argv[1],NULL); - } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { - cserver.client_max_querybuf_len = memtoll(argv[1],NULL); + loadServerConfig(argv[1], 0, NULL); } else if ((!strcasecmp(argv[0],"slaveof") || !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; @@ -559,7 +592,7 @@ void loadServerConfigFromString(char *config) { while ((ln = listNext(&li))) { struct redisMaster *mi = (struct redisMaster*)listNodeValue(ln); - zfree(mi->masterauth); + sdsfree(mi->masterauth); zfree(mi->masteruser); zfree(mi->repl_transfer_tmpfile); delete mi->staleKeyMap; @@ -580,21 +613,6 @@ void loadServerConfigFromString(char *config) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; goto loaderr; } - /* The old "requirepass" directive just translates to setting - * a password to the default user. The only thing we do - * additionally is to remember the cleartext password in this - * case, for backward compatibility with Redis <= 5. */ - ACLSetUser(DefaultUser,"resetpass",-1); - sdsfree(g_pserver->requirepass); - g_pserver->requirepass = NULL; - if (sdslen(argv[1])) { - sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); - ACLSetUser(DefaultUser,aclop,sdslen(aclop)); - sdsfree(aclop); - g_pserver->requirepass = sdsnew(argv[1]); - } else { - ACLSetUser(DefaultUser,"nopass",-1); - } } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { @@ -678,8 +696,7 @@ void loadServerConfigFromString(char *config) { err = "sentinel directive while not in sentinel mode"; goto loaderr; } - err = sentinelHandleConfiguration(argv+1,argc-1); - if (err) goto loaderr; + queueSentinelConfig(argv+1,argc-1,linenum,lines[i]); } } else if (!strcasecmp(argv[0],"scratch-file-path")) { #ifdef USE_MEMKIND @@ -753,6 +770,10 @@ void loadServerConfigFromString(char *config) { goto loaderr; } + /* To ensure backward compatibility and work while hz is out of range */ + if (g_pserver->config_hz < CONFIG_MIN_HZ) g_pserver->config_hz = CONFIG_MIN_HZ; + if (g_pserver->config_hz > CONFIG_MAX_HZ) g_pserver->config_hz = CONFIG_MAX_HZ; + sdsfreesplitres(lines,totlines); return; @@ -772,28 +793,31 @@ loaderr: * Both filename and options can be NULL, in such a case are considered * empty. This way loadServerConfig can be used to just load a file or * just load a string. */ -void loadServerConfig(char *filename, char *options) { +void loadServerConfig(char *filename, char config_from_stdin, char *options) { sds config = sdsempty(); char buf[CONFIG_MAX_LINE+1]; + FILE *fp; /* Load the file content */ if (filename) { - FILE *fp; - - if (filename[0] == '-' && filename[1] == '\0') { - fp = stdin; - } else { - if ((fp = fopen(filename,"r")) == NULL) { - serverLog(LL_WARNING, + if ((fp = fopen(filename,"r")) == NULL) { + serverLog(LL_WARNING, "Fatal error, can't open config file '%s': %s", filename, strerror(errno)); - exit(1); - } + exit(1); } while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) config = sdscat(config,buf); - if (fp != stdin) fclose(fp); + fclose(fp); } + /* Append content from stdin */ + if (config_from_stdin) { + serverLog(LL_WARNING,"Reading config from stdin"); + fp = stdin; + while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) + config = sdscat(config,buf); + } + /* Append the additional options */ if (options) { config = sdscat(config,"\n"); @@ -846,9 +870,13 @@ void configSetCommand(client *c) { /* Iterate the configs that are standard */ for (standardConfig *config = configs; config->name != NULL; config++) { - if(config->modifiable && (!strcasecmp(szFromObj(c->argv[2]),config->name) || + if (!(config->flags & IMMUTABLE_CONFIG) && + (!strcasecmp(szFromObj(c->argv[2]),config->name) || (config->alias && !strcasecmp(szFromObj(c->argv[2]),config->alias)))) { + if (config->flags & SENSITIVE_CONFIG) { + preventCommandLogging(c); + } if (!config->interface.set(config->data,szFromObj(o),1,&errstr)) { goto badfmt; } @@ -860,23 +888,38 @@ void configSetCommand(client *c) { if (0) { /* this starts the config_set macros else-if chain. */ /* Special fields that can't be handled with general macros. */ - config_set_special_field("requirepass") { - if (sdslen(szFromObj(o)) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; - /* The old "requirepass" directive just translates to setting - * a password to the default user. The only thing we do - * additionally is to remember the cleartext password in this - * case, for backward compatibility with Redis <= 5. */ - ACLSetUser(DefaultUser,"resetpass",-1); - sdsfree(g_pserver->requirepass); - g_pserver->requirepass = NULL; - if (sdslen(szFromObj(o))) { - sds aclop = sdscatprintf(sdsempty(),">%s",(char*)ptrFromObj(o)); - ACLSetUser(DefaultUser,aclop,sdslen(aclop)); - sdsfree(aclop); - g_pserver->requirepass = sdsnew(szFromObj(o)); - } else { - ACLSetUser(DefaultUser,"nopass",-1); + config_set_special_field("bind") { + int vlen; + sds *v = sdssplitlen(szFromObj(o),sdslen(szFromObj(o))," ",1,&vlen); + + if (vlen < 1 || vlen > CONFIG_BINDADDR_MAX) { + addReplyError(c, "Too many bind addresses specified."); + sdsfreesplitres(v, vlen); + return; } + + if (changeBindAddr(v, vlen, true) == C_ERR) { + addReplyError(c, "Failed to bind to specified addresses."); + sdsfreesplitres(v, vlen); + return; + } + // Now run the config change on the other threads + for (int ithread = 0; ithread < cserver.cthreads; ++ithread) { + if (&g_pserver->rgthreadvar[ithread] != serverTL) { + incrRefCount(o); + aePostFunction(g_pserver->rgthreadvar[ithread].el, [o]{ + int vlen; + sds *v = sdssplitlen(szFromObj(o),sdslen(szFromObj(o))," ",1,&vlen); + if (changeBindAddr(v, vlen, false) == C_ERR) { + serverLog(LL_WARNING, "Failed to change the bind address for a thread. Server will still be listening on old addresses."); + } + sdsfreesplitres(v, vlen); + decrRefCount(o); + }); + } + } + + sdsfreesplitres(v, vlen); } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(szFromObj(o),sdslen(szFromObj(o))," ",1,&vlen); @@ -985,10 +1028,6 @@ void configSetCommand(client *c) { enableWatchdog(ll); else disableWatchdog(); - /* Memory fields. - * config_set_memory_field(name,var) */ - } config_set_memory_field( - "client-query-buffer-limit",cserver.client_max_querybuf_len) { /* Everything else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", @@ -1023,7 +1062,7 @@ badfmt: /* Bad format errors */ addReplyBulkCString(c,_var ? _var : ""); \ matches++; \ } \ -} while(0); +} while(0) #define config_get_bool_field(_name,_var) do { \ if (stringmatch(pattern,_name,1)) { \ @@ -1031,7 +1070,7 @@ badfmt: /* Bad format errors */ addReplyBulkCString(c,_var ? "yes" : "no"); \ matches++; \ } \ -} while(0); +} while(0) #define config_get_numerical_field(_name,_var) do { \ if (stringmatch(pattern,_name,1)) { \ @@ -1040,8 +1079,7 @@ badfmt: /* Bad format errors */ addReplyBulkCString(c,buf); \ matches++; \ } \ -} while(0); - +} while(0) void configGetCommand(client *c) { robj *o = c->argv[2]; @@ -1069,7 +1107,6 @@ void configGetCommand(client *c) { config_get_string_field("logfile",g_pserver->logfile); /* Numerical values */ - config_get_numerical_field("client-query-buffer-limit",cserver.client_max_querybuf_len); config_get_numerical_field("watchdog-period",g_pserver->watchdog_period); /* Everything we can't handle with macros follows. */ @@ -1120,7 +1157,7 @@ void configGetCommand(client *c) { } if (stringmatch(pattern,"unixsocketperm",1)) { char buf[32]; - snprintf(buf,sizeof(buf),"%o",g_pserver->unixsocketperm); + snprintf(buf,sizeof(buf),"%lo",(unsigned long)g_pserver->unixsocketperm); addReplyBulkCString(c,"unixsocketperm"); addReplyBulkCString(c,buf); matches++; @@ -1174,16 +1211,6 @@ void configGetCommand(client *c) { sdsfree(aux); matches++; } - if (stringmatch(pattern,"requirepass",1)) { - addReplyBulkCString(c,"requirepass"); - sds password = g_pserver->requirepass; - if (password) { - addReplyBulkCBuffer(c,password,sdslen(password)); - } else { - addReplyBulkCString(c,""); - } - matches++; - } if (stringmatch(pattern,"oom-score-adj-values",0)) { sds buf = sdsempty(); int j; @@ -1232,7 +1259,8 @@ dictType optionToLineDictType = { NULL, /* val dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ - dictListDestructor /* val destructor */ + dictListDestructor, /* val destructor */ + NULL /* allow to expand */ }; dictType optionSetDictType = { @@ -1241,7 +1269,8 @@ dictType optionSetDictType = { NULL, /* val dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ - NULL /* val destructor */ + NULL, /* val destructor */ + NULL /* allow to expand */ }; /* The config rewrite state. */ @@ -1345,13 +1374,22 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { char *p = strstr(argv[0],"slave"); if (p) { sds alt = sdsempty(); - alt = sdscatlen(alt,argv[0],p-argv[0]);; + alt = sdscatlen(alt,argv[0],p-argv[0]); alt = sdscatlen(alt,"replica",7); alt = sdscatlen(alt,p+5,strlen(p+5)); sdsfree(argv[0]); argv[0] = alt; } - rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + /* If this is sentinel config, we use sentinel "sentinel " as option + to avoid messing up the sequence. */ + if (g_pserver->sentinel_mode && argc > 1 && !strcasecmp(argv[0],"sentinel")) { + sds sentinelOption = sdsempty(); + sentinelOption = sdscatfmt(sentinelOption,"%S %S",argv[0],argv[1]); + rewriteConfigAddLineNumberToOption(state,sentinelOption,linenum); + sdsfree(sentinelOption); + } else { + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + } sdsfreesplitres(argv,argc); } fclose(fp); @@ -1469,6 +1507,28 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, const char *opt rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite a SDS string option. */ +void rewriteConfigSdsOption(struct rewriteConfigState *state, const char *option, sds value, const sds defvalue) { + int force = 1; + sds line; + + /* If there is no value set, we don't want the SDS option + * to be present in the configuration at all. */ + if (value == NULL) { + rewriteConfigMarkAsProcessed(state, option); + return; + } + + /* Set force to zero if the value is set to its default. */ + if (defvalue && sdscmp(value, defvalue) == 0) force = 0; + + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatrepr(line, value, sdslen(value)); + + rewriteConfigRewriteLine(state, option, line, force); +} + /* Rewrite a numerical (long long range) option. */ void rewriteConfigNumericalOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { int force = value != defvalue; @@ -1508,15 +1568,20 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { return; } - /* Note that if there are no save parameters at all, all the current - * config line with "save" will be detected as orphaned and deleted, - * resulting into no RDB persistence as expected. */ - for (j = 0; j < g_pserver->saveparamslen; j++) { - line = sdscatprintf(sdsempty(),"save %ld %d", - (long) g_pserver->saveparams[j].seconds, g_pserver->saveparams[j].changes); - rewriteConfigRewriteLine(state,"save",line,1); + /* Rewrite save parameters, or an empty 'save ""' line to avoid the + * defaults from being used. + */ + if (!g_pserver->saveparamslen) { + rewriteConfigRewriteLine(state,"save",sdsnew("save \"\""),1); + } else { + for (j = 0; j < g_pserver->saveparamslen; j++) { + line = sdscatprintf(sdsempty(),"save %ld %d", + (long) g_pserver->saveparams[j].seconds, g_pserver->saveparams[j].changes); + rewriteConfigRewriteLine(state,"save",line,1); + } } - /* Mark "save" as processed in case g_pserver->saveparamslen is zero. */ + + /* Mark "save" as processed in case server.saveparamslen is zero. */ rewriteConfigMarkAsProcessed(state,"save"); } @@ -1672,26 +1737,6 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,force); } -/* Rewrite the requirepass option. */ -void rewriteConfigRequirepassOption(struct rewriteConfigState *state, const char *option) { - int force = 1; - sds line; - sds password = g_pserver->requirepass; - - /* If there is no password set, we don't want the requirepass option - * to be present in the configuration at all. */ - if (password == NULL) { - rewriteConfigMarkAsProcessed(state,option); - return; - } - - line = sdsnew(option); - line = sdscatlen(line, " ", 1); - line = sdscatsds(line, password); - - rewriteConfigRewriteLine(state,option,line,force); -} - /* Glue together the configuration lines in the current configuration * rewrite state into a single string, stripping multiple empty lines. */ sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { @@ -1799,7 +1844,7 @@ int rewriteConfigOverwriteFile(char *configfile, sds content) { if (fsync(fd)) serverLog(LL_WARNING, "Could not sync tmp config file to disk (%s)", strerror(errno)); - else if (fchmod(fd, 0644) == -1) + else if (fchmod(fd, 0644 & ~g_pserver->umask) == -1) serverLog(LL_WARNING, "Could not chmod config file (%s)", strerror(errno)); else if (rename(tmp_conffile, configfile) == -1) serverLog(LL_WARNING, "Could not rename tmp config file (%s)", strerror(errno)); @@ -1848,8 +1893,6 @@ int rewriteConfig(char *path, int force_all) { rewriteConfigUserOption(state); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state,"replicaof"); - rewriteConfigRequirepassOption(state,"requirepass"); - rewriteConfigBytesOption(state,"client-query-buffer-limit",cserver.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); rewriteConfigStringOption(state,"cluster-config-file",g_pserver->cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigClientoutputbufferlimitOption(state); @@ -1882,14 +1925,11 @@ int rewriteConfig(char *path, int force_all) { #define LOADBUF_SIZE 256 static char loadbuf[LOADBUF_SIZE]; -#define MODIFIABLE_CONFIG 1 -#define IMMUTABLE_CONFIG 0 - -#define embedCommonConfig(config_name, config_alias, is_modifiable) \ - config_name, config_alias, is_modifiable, +#define embedCommonConfig(config_name, config_alias, config_flags) \ + config_name, config_alias, config_flags, #define embedConfigInterface(initfn, setfn, getfn, rewritefn) { \ - initfn, nullptr, setfn, getfn, rewritefn, \ + initfn, setfn, getfn, rewritefn, \ }, /* What follows is the generic config types that are supported. To add a new @@ -1936,11 +1976,11 @@ static void boolConfigRewrite(typeData data, const char *name, struct rewriteCon rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); } -constexpr standardConfig createBoolConfig(const char *name, const char *alias, int modifiable, int &config_addr, int defaultValue, int (*is_valid)(int val, const char **err), int (*update)(int val, int prev, const char **err)) +constexpr standardConfig createBoolConfig(const char *name, const char *alias, unsigned flags, int &config_addr, int defaultValue, int (*is_valid)(int val, const char **err), int (*update)(int val, int prev, const char **err)) { standardConfig conf = { - embedCommonConfig(name, alias, modifiable) - { boolConfigInit, nullptr, boolConfigSet, boolConfigGet, boolConfigRewrite } + embedCommonConfig(name, alias, flags) + { boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite } }; conf.data.yesno.config = &config_addr; conf.data.yesno.default_value = defaultValue; @@ -1951,22 +1991,14 @@ constexpr standardConfig createBoolConfig(const char *name, const char *alias, i /* String Configs */ static void stringConfigInit(typeData data) { - if (data.string.convert_empty_to_null) { - *data.string.config = data.string.default_value ? zstrdup(data.string.default_value) : NULL; - } else { - *data.string.config = zstrdup(data.string.default_value); - } + *data.string.config = (data.string.convert_empty_to_null && !data.string.default_value) ? NULL : zstrdup(data.string.default_value); } static int stringConfigSet(typeData data, sds value, int update, const char **err) { if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) return 0; char *prev = *data.string.config; - if (data.string.convert_empty_to_null) { - *data.string.config = value[0] ? zstrdup(value) : NULL; - } else { - *data.string.config = zstrdup(value); - } + *data.string.config = (data.string.convert_empty_to_null && !value[0]) ? NULL : zstrdup(value); if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { zfree(*data.string.config); *data.string.config = prev; @@ -1984,12 +2016,44 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value); } +/* SDS Configs */ +static void sdsConfigInit(typeData data) { + *data.sds.config = (data.sds.convert_empty_to_null && !data.sds.default_value) ? NULL: sdsnew(data.sds.default_value); +} + +static int sdsConfigSet(typeData data, sds value, int update, const char **err) { + if (data.sds.is_valid_fn && !data.sds.is_valid_fn(value, err)) + return 0; + sds prev = *data.sds.config; + *data.sds.config = (data.sds.convert_empty_to_null && (sdslen(value) == 0)) ? NULL : sdsdup(value); + if (update && data.sds.update_fn && !data.sds.update_fn(*data.sds.config, prev, err)) { + sdsfree(*data.sds.config); + *data.sds.config = prev; + return 0; + } + sdsfree(prev); + return 1; +} + +static void sdsConfigGet(client *c, typeData data) { + if (*data.sds.config) { + addReplyBulkSds(c, sdsdup(*data.sds.config)); + } else { + addReplyBulkCString(c, ""); + } +} + +static void sdsConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigSdsOption(state, name, *(data.sds.config), data.sds.default_value ? sdsnew(data.sds.default_value) : NULL); +} + + #define ALLOW_EMPTY_STRING 0 #define EMPTY_STRING_IS_NULL 1 -constexpr standardConfig createStringConfig(const char *name, const char *alias, int modifiable, int empty_to_null, char *&config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { +constexpr standardConfig createStringConfig(const char *name, const char *alias, unsigned flags, int empty_to_null, char *&config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { standardConfig conf = { - embedCommonConfig(name, alias, modifiable) + embedCommonConfig(name, alias, flags) embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) }; conf.data.string = { @@ -2002,6 +2066,21 @@ constexpr standardConfig createStringConfig(const char *name, const char *alias, return conf; } +constexpr standardConfig createSDSConfig(const char *name, const char *alias, unsigned flags, int empty_to_null, sds &config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { + standardConfig conf = { + embedCommonConfig(name, alias, flags) + embedConfigInterface(sdsConfigInit, sdsConfigSet, sdsConfigGet, sdsConfigRewrite) + }; + conf.data.sds = { + &(config_addr), + (defaultValue), + (is_valid), + (update), + (empty_to_null), + }; + return conf; +} + /* Enum configs */ static void enumConfigInit(typeData data) { *data.enumd.config = data.enumd.default_value; @@ -2020,7 +2099,7 @@ static int enumConfigSet(typeData data, sds value, int update, const char **err) } sdsrange(enumerr,0,-3); /* Remove final ", ". */ - strncpy(loadbuf, enumerr, LOADBUF_SIZE-1); + strncpy(loadbuf, enumerr, LOADBUF_SIZE); loadbuf[LOADBUF_SIZE - 1] = '\0'; sdsfree(enumerr); @@ -2046,9 +2125,9 @@ static void enumConfigRewrite(typeData data, const char *name, struct rewriteCon rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); } -constexpr standardConfig createEnumConfig(const char *name, const char *alias, int modifiable, configEnum *enumVal, int &config_addr, int defaultValue, int (*is_valid)(int,const char**), int (*update)(int,int,const char**)) { +constexpr standardConfig createEnumConfig(const char *name, const char *alias, unsigned flags, configEnum *enumVal, int &config_addr, int defaultValue, int (*is_valid)(int,const char**), int (*update)(int,int,const char**)) { standardConfig c = { - embedCommonConfig(name, alias, modifiable) + embedCommonConfig(name, alias, flags) embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) }; c.data.enumd = { @@ -2204,9 +2283,9 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite #define INTEGER_CONFIG 0 #define MEMORY_CONFIG 1 -constexpr standardConfig embedCommonNumericalConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { +constexpr standardConfig embedCommonNumericalConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { standardConfig conf = { - embedCommonConfig(name, alias, modifiable) + embedCommonConfig(name, alias, flags) embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) }; conf.data.numeric.is_memory = (memory); @@ -2218,73 +2297,73 @@ constexpr standardConfig embedCommonNumericalConfig(const char *name, const char return conf; } -constexpr standardConfig createIntConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) +constexpr standardConfig createIntConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - standardConfig conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); + standardConfig conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_INT; conf.data.numeric.config.i = &config_addr; return conf; } -constexpr standardConfig createUIntConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, unsigned int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) +constexpr standardConfig createUIntConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, unsigned int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_UINT; conf.data.numeric.config.ui = &(config_addr); return conf; } -constexpr standardConfig createLongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createLongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_LONG; conf.data.numeric.config.l = &(config_addr); return conf; } -constexpr standardConfig createULongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, unsigned long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createULongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, unsigned long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_ULONG; conf.data.numeric.config.ul = &(config_addr); return conf; } -constexpr standardConfig createLongLongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createLongLongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_LONG_LONG; conf.data.numeric.config.ll = &(config_addr); return conf; } -constexpr standardConfig createULongLongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, unsigned long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createULongLongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, unsigned long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_ULONG_LONG; conf.data.numeric.config.ull = &(config_addr); return conf; } -constexpr standardConfig createSizeTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, size_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createSizeTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, size_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_SIZE_T; conf.data.numeric.config.st = &(config_addr); return conf; } -constexpr standardConfig createSSizeTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, ssize_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createSSizeTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, ssize_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_SSIZE_T; conf.data.numeric.config.sst = &(config_addr); return conf; } -constexpr standardConfig createTimeTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, time_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createTimeTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, time_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_TIME_T; conf.data.numeric.config.tt = &(config_addr); return conf; } -constexpr standardConfig createOffTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, off_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createOffTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, off_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_OFF_T; conf.data.numeric.config.ot = &(config_addr); return conf; @@ -2294,8 +2373,8 @@ static int isValidActiveDefrag(int val, const char **err) { #ifndef HAVE_DEFRAG if (val) { *err = "Active defragmentation cannot be enabled: it " - "requires a Redis server compiled with a modified Jemalloc " - "like the one shipped by default with the Redis source " + "requires a KeyDB server compiled with a modified Jemalloc " + "like the one shipped by default with the KeyDB source " "distribution"; return 0; } @@ -2322,6 +2401,25 @@ static int isValidAOFfilename(char *val, const char **err) { return 1; } +/* Validate specified string is a valid proc-title-template */ +static int isValidProcTitleTemplate(char *val, const char **err) { + if (!validateProcTitleTemplate(val)) { + *err = "template format is invalid or contains unknown variables"; + return 0; + } + return 1; +} + +static int updateProcTitleTemplate(char *val, char *prev, const char **err) { + UNUSED(val); + UNUSED(prev); + if (redisSetProcTitle(NULL) == C_ERR) { + *err = "failed to set process title"; + return 0; + } + return 1; +} + static int updateHZ(long long val, long long prev, const char **err) { UNUSED(prev); UNUSED(err); @@ -2334,6 +2432,32 @@ static int updateHZ(long long val, long long prev, const char **err) { return 1; } +static int updatePort(long long val, long long prev, const char **err) { + /* Do nothing if port is unchanged */ + if (val == prev) { + return 1; + } + + // Run this thread to make sure its valid + if (changeListenPort(val, &serverTL->ipfd, acceptTcpHandler, true) == C_ERR) { + *err = "Unable to listen on this port. Check server logs."; + return 0; + } + + // Now run the config change on the other threads + for (int ithread = 0; ithread < cserver.cthreads; ++ithread) { + if (&g_pserver->rgthreadvar[ithread] != serverTL) { + aePostFunction(g_pserver->rgthreadvar[ithread].el, [val]{ + if (changeListenPort(val, &serverTL->ipfd, acceptTcpHandler, false) == C_ERR) { + serverLog(LL_WARNING, "Failed to change the listen port for a thread. Server will still be listening on old ports."); + } + }); + } + } + + return 1; +} + static int updateJemallocBgThread(int val, int prev, const char **err) { UNUSED(prev); UNUSED(err); @@ -2358,7 +2482,7 @@ static int updateMaxmemory(long long val, long long prev, const char **err) { if ((unsigned long long)val < used) { serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", g_pserver->maxmemory, used); } - freeMemoryIfNeededAndSafe(false /*fQuickCycle*/, false /*fPreSnapshot*/); + performEvictions(false /*fPreSnapshot*/); } return 1; } @@ -2371,7 +2495,7 @@ static int updateGoodSlaves(long long val, long long prev, const char **err) { return 1; } -static int updateMasterAuthConfig(char *, char *, const char **) { +static int updateMasterAuthConfig(sds, sds, const char **) { updateMasterAuth(); return 1; } @@ -2389,6 +2513,16 @@ static int updateAppendonly(int val, int prev, const char **err) { return 1; } +static int updateSighandlerEnabled(int val, int prev, const char **err) { + UNUSED(err); + UNUSED(prev); + if (val) + setupSignalHandlers(); + else + removeSignalHandlers(); + return 1; +} + static int updateMaxclients(long long val, long long prev, const char **err) { /* Try to check if the OS is capable of supporting so many FDs. */ if (val > prev) { @@ -2411,7 +2545,7 @@ static int updateMaxclients(long long val, long long prev, const char **err) { if (aeResizeSetSize(aeGetCurrentEventLoop(), g_pserver->maxclients + CONFIG_FDSET_INCR) == AE_ERR) { - *err = "The event loop API used by Redis is not able to handle the specified number of clients"; + *err = "The event loop API used by KeyDB is not able to handle the specified number of clients"; return 0; } serverLog(LL_DEBUG,"Successfully changed the setsize for current thread %d", ielFromEventLoop(aeGetCurrentEventLoop())); @@ -2465,6 +2599,17 @@ static int updateOOMScoreAdj(int val, int prev, const char **err) { return 1; } +int updateRequirePass(sds val, sds prev, const char **err) { + UNUSED(prev); + UNUSED(err); + /* The old "requirepass" directive just translates to setting + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ + ACLUpdateDefaultUserPassword(val); + return 1; +} + #ifdef USE_OPENSSL static int updateTlsCfg(char *val, char *prev, const char **err) { UNUSED(val); @@ -2490,6 +2635,48 @@ static int updateTlsCfgInt(long long val, long long prev, const char **err) { UNUSED(prev); return updateTlsCfg(NULL, NULL, err); } + +static int updateTLSPortThread(long long val, bool fFirstCall, const char **err) +{ + if (changeListenPort(val, &serverTL->tlsfd, acceptTLSHandler, fFirstCall) == C_ERR) { + *err = "Unable to listen on this port. Check server logs."; + return 0; + } + + return 1; +} + +static int updateTLSPort(long long val, long long prev, const char **err) { + /* Do nothing if port is unchanged */ + if (val == prev) { + return 1; + } + + /* Configure TLS if tls is enabled */ + if (prev == 0 && tlsConfigure(&g_pserver->tls_ctx_config) == C_ERR) { + *err = "Unable to update TLS configuration. Check server logs."; + return 0; + } + + // Do our thread first in case there is a config issue + if (!updateTLSPortThread(val, true /*fFirstCall*/, err)) + return 0; + + for (int ithread = 0; ithread < cserver.cthreads; ++ithread) { + if (ithread == serverTL - g_pserver->rgthreadvar) + continue; // we already did our thread + aePostFunction(g_pserver->rgthreadvar[ithread].el, [val]{ + const char **err = nullptr; + if (!updateTLSPortThread(val, false /*fFirstCall*/, err)) { + serverLog(LL_WARNING, "Failed to update TLS port for a thread: %s", *err); + serverLog(LL_WARNING, "\tKeyDB will still be listening on the old port for some threads."); + } + }); + } + + return 1; +} + #endif /* USE_OPENSSL */ int fDummy = false; @@ -2505,11 +2692,13 @@ standardConfig configs[] = { createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, g_pserver->rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, g_pserver->activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, g_pserver->stop_writes_on_bgsave_err, 1, NULL, NULL), + createBoolConfig("set-proc-title", NULL, IMMUTABLE_CONFIG, cserver.set_proc_title, 1, NULL, NULL), /* Should setproctitle be used? */ createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, g_pserver->dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_eviction, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_expire, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_server_del, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_user_del , 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-user-flush", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_user_flush , 0, NULL, NULL), createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, g_pserver->repl_disable_tcp_nodelay, 0, NULL, NULL), createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, g_pserver->repl_diskless_sync, 0, NULL, NULL), createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, g_pserver->aof_rewrite_incremental_fsync, 1, NULL, NULL), @@ -2531,19 +2720,22 @@ standardConfig configs[] = { createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, g_pserver->aof_enabled, 0, NULL, updateAppendonly), createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_reads_when_down, 0, NULL, NULL), createBoolConfig("delete-on-evict", NULL, MODIFIABLE_CONFIG, cserver.delete_on_evict, 0, NULL, NULL), - createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL), createBoolConfig("use-fork", NULL, IMMUTABLE_CONFIG, cserver.fForkBgSave, 0, NULL, NULL), - createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL), createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, fDummy, 0, NULL, NULL), createBoolConfig("prefetch-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->prefetch_enabled, 1, NULL, NULL), + createBoolConfig("crash-log-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->crashlog_enabled, 1, NULL, updateSighandlerEnabled), + createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->memcheck_enabled, 1, NULL, NULL), + createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, g_pserver->use_exit_on_panic, 0, NULL, NULL), + createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, g_pserver->disable_thp, 1, NULL, NULL), + createBoolConfig("cluster-allow-replica-migration", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_replica_migration, 1, NULL, NULL), + createBoolConfig("replica-announced", NULL, MODIFIABLE_CONFIG, g_pserver->replica_announced, 1, NULL, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL), createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->unixsocket, NULL, NULL, NULL), createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.pidfile, NULL, NULL, NULL), createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->slave_announce_ip, NULL, NULL, NULL), - createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masteruser, NULL, NULL, updateMasterAuthConfig), - createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masterauth, NULL, NULL, updateMasterAuthConfig), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masteruser, NULL, NULL, updateMasterAuthConfig), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->cluster_announce_ip, NULL, NULL, NULL), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->syslog_ident, "redis", NULL, NULL), createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->rdb_filename, CONFIG_DEFAULT_RDB_FILENAME, isValidDBfilename, NULL), @@ -2553,7 +2745,12 @@ standardConfig configs[] = { createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->aof_rewrite_cpulist, NULL, NULL, NULL), createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->bgsave_cpulist, NULL, NULL, NULL), createStringConfig("storage-provider-options", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.storage_conf, NULL, NULL, NULL), - createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->ignore_warnings, "ARM64-COW-BUG", NULL, NULL), + createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->ignore_warnings, "", NULL, NULL), + createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, cserver.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate), + + /* SDS Configs */ + createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masterauth, NULL, NULL, updateMasterAuthConfig), + createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->requirepass, NULL, NULL, updateRequirePass), /* Enum Configs */ createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, cserver.supervised_mode, SUPERVISED_NONE, NULL, NULL), @@ -2564,10 +2761,12 @@ standardConfig configs[] = { createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, g_pserver->aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), createEnumConfig("storage-cache-mode", NULL, IMMUTABLE_CONFIG, storage_memory_model_enum, cserver.storage_memory_model, STORAGE_WRITETHROUGH, NULL, NULL), createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, g_pserver->oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj), + createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, g_pserver->acl_pubsub_default, USER_FLAG_ALLCHANNELS, NULL, NULL), + createEnumConfig("sanitize-dump-payload", NULL, MODIFIABLE_CONFIG, sanitize_dump_payload_enum, cserver.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL), /* Integer configs */ createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, cserver.dbnum, 16, INTEGER_CONFIG, NULL, NULL), - createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, g_pserver->port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createIntConfig("port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->port, 6379, INTEGER_CONFIG, NULL, updatePort), /* TCP port. */ createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL), createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */ createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, g_pserver->list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL), @@ -2582,16 +2781,18 @@ standardConfig configs[] = { createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->slave_priority, 100, INTEGER_CONFIG, NULL, NULL), createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL), createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->maxmemory_samples, 16, INTEGER_CONFIG, NULL, NULL), + createIntConfig("maxmemory-eviction-tenacity", NULL, MODIFIABLE_CONFIG, 0, 100, g_pserver->maxmemory_eviction_tenacity, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, cserver.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */ createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, g_pserver->slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, g_pserver->tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use g_pserver->port */ + createIntConfig("cluster-announce-tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.tls_port */ createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, g_pserver->rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, g_pserver->key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, cserver.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), @@ -2604,7 +2805,6 @@ standardConfig configs[] = { createUIntConfig("loading-process-events-interval-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_keys, 8192, MEMORY_CONFIG, NULL, NULL), /* Unsigned Long configs */ - createULongConfig("loading-process-events-interval-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_bytes, 2*1024*1024, MEMORY_CONFIG, NULL, NULL), createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, cserver.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), @@ -2632,22 +2832,32 @@ standardConfig configs[] = { createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */ + createSizeTConfig("client-query-buffer-limit", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, cserver.client_max_querybuf_len, 1024*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Default: 1GB max query buffer. */ /* Other configs */ createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL), + /* KeyDB Specific Configs */ + createULongConfig("loading-process-events-interval-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_bytes, 2*1024*1024, MEMORY_CONFIG, NULL, NULL), + createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL), + createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL), + #ifdef USE_OPENSSL - createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */ createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt), createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt), - createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, g_pserver->tls_cluster, 0, NULL, NULL), - createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, g_pserver->tls_replication, 0, NULL, NULL), + createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, g_pserver->tls_cluster, 0, NULL, updateTlsCfgBool), + createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, g_pserver->tls_replication, 0, NULL, updateTlsCfgBool), createEnumConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, tls_auth_clients_enum, g_pserver->tls_auth_clients, TLS_CLIENT_AUTH_YES, NULL, NULL), createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, g_pserver->tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool), createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, g_pserver->tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool), createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.key_file_pass, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.client_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.client_key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.client_key_file_pass, NULL, NULL, updateTlsCfg), createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), @@ -2673,12 +2883,17 @@ void configCommand(client *c) { if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"help")) { const char *help[] = { -"GET -- Return parameters matching the glob-like and their values.", -"SET -- Set parameter to value.", -"RESETSTAT -- Reset statistics reported by INFO.", -"REWRITE -- Rewrite the configuration file.", +"GET ", +" Return parameters matching the glob-like and their values.", +"SET ", +" Set the configuration to .", +"RESETSTAT", +" Reset statistics reported by the INFO command.", +"REWRITE", +" Rewrite the configuration file.", NULL }; + addReplyHelp(c, help); } else if (!strcasecmp(szFromObj(c->argv[1]),"set") && c->argc == 4) { configSetCommand(c); @@ -2687,6 +2902,7 @@ NULL } else if (!strcasecmp(szFromObj(c->argv[1]),"resetstat") && c->argc == 2) { resetServerStats(); resetCommandTableStats(); + resetErrorTableStats(); addReply(c,shared.ok); } else if (!strcasecmp(szFromObj(c->argv[1]),"rewrite") && c->argc == 2) { if (cserver.configfile == NULL) { diff --git a/src/config.h b/src/config.h index 51fe6e253..addb9c3f4 100644 --- a/src/config.h +++ b/src/config.h @@ -35,10 +35,11 @@ #endif #ifdef __linux__ -#include #include #endif +#define CONFIG_DEFAULT_RDB_FILENAME "dump.rdb" + /* Define redis_fstat to fstat or fstat64() */ #if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) #define redis_fstat fstat64 @@ -65,7 +66,7 @@ /* Test for backtrace() */ #if defined(__APPLE__) || (defined(__linux__) && defined(__GLIBC__)) || \ defined(__FreeBSD__) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(USE_BACKTRACE))\ - || defined(__DragonFly__) + || defined(__DragonFly__) || (defined(__UCLIBC__) && defined(__UCLIBC_HAS_BACKTRACE__)) #define HAVE_BACKTRACE 1 #endif @@ -87,6 +88,7 @@ #include #ifdef _DTRACE_VERSION #define HAVE_EVPORT 1 +#define HAVE_PSINFO 1 #endif #endif @@ -97,21 +99,23 @@ #define redis_fsync fsync #endif -/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use - * the plain fsync() call. */ -#ifdef __linux__ -#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) -#if (LINUX_VERSION_CODE >= 0x020611 && __GLIBC_PREREQ(2, 6)) -#define HAVE_SYNC_FILE_RANGE 1 -#endif +#if __GNUC__ >= 4 +#define redis_unreachable __builtin_unreachable #else -#if (LINUX_VERSION_CODE >= 0x020611) -#define HAVE_SYNC_FILE_RANGE 1 -#endif -#endif +#define redis_unreachable abort #endif -#ifdef HAVE_SYNC_FILE_RANGE +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use + * the plain fsync() call. */ +#if (defined(__linux__) && defined(SYNC_FILE_RANGE_WAIT_BEFORE)) #define rdb_fsync_range(fd,off,size) sync_file_range(fd,off,size,SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE) #else #define rdb_fsync_range(fd,off,size) fsync(fd) @@ -128,7 +132,7 @@ #define ESOCKTNOSUPPORT 0 #endif -#if ((defined __linux && defined(__GLIBC__)) || defined __APPLE__) +#if (defined __linux || defined __APPLE__) #define USE_SETPROCTITLE #define INIT_SETPROCTITLE_REPLACEMENT #ifdef __cplusplus @@ -247,12 +251,12 @@ void setproctitle(const char *fmt, ...); #elif defined __NetBSD__ #include #define redis_set_thread_title(name) pthread_setname_np(pthread_self(), "%s", name) +#elif defined __HAIKU__ +#include +#define redis_set_thread_title(name) rename_thread(find_thread(0), name) #else #if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7)) -#ifdef __cplusplus -extern "C" -#endif -int pthread_setname_np(const char *name); +extern "C" int pthread_setname_np(const char *name); #include #define redis_set_thread_title(name) pthread_setname_np(name) #else diff --git a/src/connection.cpp b/src/connection.cpp index 4e6133350..be024dff1 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -148,8 +148,7 @@ void *connGetPrivateData(connection *conn) { /* Close the connection and free resources. */ static void connSocketClose(connection *conn) { if (conn->fd != -1) { - aeDeleteFileEvent(serverTL->el,conn->fd,AE_READABLE); - aeDeleteFileEvent(serverTL->el,conn->fd,AE_WRITABLE); + aeDeleteFileEvent(serverTL->el,conn->fd,AE_READABLE | AE_WRITABLE); close(conn->fd); conn->fd = -1; } @@ -403,15 +402,15 @@ int connGetSocketError(connection *conn) { } int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port) { - return anetPeerToString(conn ? conn->fd : -1, ip, ip_len, port); -} - -int connFormatPeer(connection *conn, char *buf, size_t buf_len) { - return anetFormatPeer(conn ? conn->fd : -1, buf, buf_len); + return anetFdToString(conn ? conn->fd : -1, ip, ip_len, port, FD_TO_PEER_NAME); } int connSockName(connection *conn, char *ip, size_t ip_len, int *port) { - return anetSockName(conn->fd, ip, ip_len, port); + return anetFdToString(conn->fd, ip, ip_len, port, FD_TO_SOCK_NAME); +} + +int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type) { + return anetFormatFdAddr(conn ? conn->fd : -1, buf, buf_len, fd_to_str_type); } int connBlock(connection *conn) { @@ -466,6 +465,6 @@ void connSetThreadAffinity(connection *conn, int cpu) { * For sockets, we always return "fd=" to maintain compatibility. */ const char *connGetInfo(connection *conn, char *buf, size_t buf_len) { - snprintf(buf, buf_len-1, "fd=%i", conn->fd); + snprintf(buf, buf_len-1, "fd=%i", conn == NULL ? -1 : conn->fd); return buf; } diff --git a/src/connection.h b/src/connection.h index 606137229..bffde3f5c 100644 --- a/src/connection.h +++ b/src/connection.h @@ -236,7 +236,7 @@ int connKeepAlive(connection *conn, int interval); int connSendTimeout(connection *conn, long long ms); int connRecvTimeout(connection *conn, long long ms); int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port); -int connFormatPeer(connection *conn, char *buf, size_t buf_len); +int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type); int connSockName(connection *conn, char *ip, size_t ip_len, int *port); const char *connGetInfo(connection *conn, char *buf, size_t buf_len); diff --git a/src/crc64.c b/src/crc64.c index 4cbc019f6..d4db4158e 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -127,14 +127,15 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { #include #define UNUSED(x) (void)(x) -int crc64Test(int argc, char *argv[]) { +int crc64Test(int argc, char *argv[], int accurate) { UNUSED(argc); UNUSED(argv); + UNUSED(accurate); crc64_init(); printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", (uint64_t)_crc64(0, "123456789", 9)); printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", - (uint64_t)crc64(0, "123456789", 9)); + (uint64_t)crc64(0, (unsigned char*)"123456789", 9)); char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " "do eiusmod tempor incididunt ut labore et dolore magna " "aliqua. Ut enim ad minim veniam, quis nostrud exercitation " @@ -146,7 +147,7 @@ int crc64Test(int argc, char *argv[]) { printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n", (uint64_t)_crc64(0, li, sizeof(li))); printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n", - (uint64_t)crc64(0, li, sizeof(li))); + (uint64_t)crc64(0, (unsigned char*)li, sizeof(li))); return 0; } diff --git a/src/crc64.h b/src/crc64.h index 08ac9f7b2..a0bf15b99 100644 --- a/src/crc64.h +++ b/src/crc64.h @@ -11,7 +11,7 @@ void crc64_init(void); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); #ifdef REDIS_TEST -int crc64Test(int argc, char *argv[]); +int crc64Test(int argc, char *argv[], int accurate); #endif #ifdef __cplusplus diff --git a/src/crcspeed.c b/src/crcspeed.c index 81a80ce8e..67cb8fd9f 100644 --- a/src/crcspeed.c +++ b/src/crcspeed.c @@ -85,7 +85,7 @@ static inline uint64_t rev8(uint64_t a) { /* This function is called once to initialize the CRC table for use on a big-endian architecture. */ void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) { - /* Create the little endian table then reverse all the entires. */ + /* Create the little endian table then reverse all the entries. */ crcspeed64little_init(fn, big_table); for (int k = 0; k < 8; k++) { for (int n = 0; n < 256; n++) { @@ -95,7 +95,7 @@ void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) { } void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) { - /* Create the little endian table then reverse all the entires. */ + /* Create the little endian table then reverse all the entries. */ crcspeed16little_init(fn, big_table); for (int k = 0; k < 8; k++) { for (int n = 0; n < 256; n++) { diff --git a/src/cron.cpp b/src/cron.cpp index 876c69d88..9ed57bae8 100644 --- a/src/cron.cpp +++ b/src/cron.cpp @@ -83,7 +83,7 @@ void executeCronJobExpireHook(const char *key, robj *o) client *cFake = createClient(nullptr, IDX_EVENT_LOOP_MAIN); cFake->lock.lock(); cFake->authenticated = 1; - cFake->puser = nullptr; + cFake->user = nullptr; selectDb(cFake, job->dbNum); serverAssert(cFake->argc == 0); diff --git a/src/db.cpp b/src/db.cpp index 77d3c1a66..71cb27a03 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -162,9 +162,8 @@ robj_roptr lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { serverAssert(GlobalLocksAcquired()); if (expireIfNeeded(db,key) == 1) { - /* Key expired. If we are in the context of a master, expireIfNeeded() - * returns 0 only when the key does not exist at all, so it's safe - * to return NULL ASAP. */ + /* If we are in the context of a master, expireIfNeeded() returns 1 + * when the key is no longer valid, so we can return NULL ASAP. */ if (listLength(g_pserver->masters) == 0) goto keymiss; @@ -196,9 +195,9 @@ robj_roptr lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { keymiss: if (!(flags & LOOKUP_NONOTIFY)) { - g_pserver->stat_keyspace_misses++; notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id); } + g_pserver->stat_keyspace_misses++; return NULL; } @@ -222,26 +221,33 @@ robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { robj *lookupKeyWrite(redisDb *db, robj *key) { return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); } - +static void SentReplyOnKeyMiss(client *c, robj *reply){ + serverAssert(sdsEncodedObject(reply)); + sds rep = szFromObj(reply); + if (sdslen(rep) > 1 && rep[0] == '-'){ + addReplyErrorObject(c, reply); + } else { + addReply(c,reply); + } +} robj_roptr lookupKeyReadOrReply(client *c, robj *key, robj *reply) { robj_roptr o = lookupKeyRead(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) { robj *o = lookupKeyWrite(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } -bool dbAddCore(redisDb *db, robj *key, robj *val, bool fUpdateMvcc, bool fAssumeNew = false) { +bool dbAddCore(redisDb *db, sds key, robj *val, bool fUpdateMvcc, bool fAssumeNew = false) { serverAssert(!val->FExpires()); - sds copy = sdsdupshared(szFromObj(key)); + sds copy = sdsdupshared(key); uint64_t mvcc = getMvccTstamp(); if (fUpdateMvcc) { - setMvccTstamp(key, mvcc); setMvccTstamp(val, mvcc); } @@ -249,11 +255,8 @@ bool dbAddCore(redisDb *db, robj *key, robj *val, bool fUpdateMvcc, bool fAssume if (fInserted) { - if (val->type == OBJ_LIST || - val->type == OBJ_ZSET || - val->type == OBJ_STREAM) - signalKeyAsReady(db, key); - if (g_pserver->cluster_enabled) slotToKeyAdd(szFromObj(key)); + signalKeyAsReady(db, key, val->type); + if (g_pserver->cluster_enabled) slotToKeyAdd(key); } else { @@ -269,13 +272,16 @@ bool dbAddCore(redisDb *db, robj *key, robj *val, bool fUpdateMvcc, bool fAssume * The program is aborted if the key already exists. */ void dbAdd(redisDb *db, robj *key, robj *val) { - bool fInserted = dbAddCore(db, key, val, true /* fUpdateMvcc */); + bool fInserted = dbAddCore(db, szFromObj(key), val, true /* fUpdateMvcc */); serverAssertWithInfo(NULL,key,fInserted); } -void redisDb::dbOverwriteCore(redisDb::iter itr, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire) +void redisDb::dbOverwriteCore(redisDb::iter itr, sds keySds, robj *val, bool fUpdateMvcc, bool fRemoveExpire) { robj *old = itr.val(); + redisObjectStack keyO; + initStaticStringObject(keyO, keySds); + robj *key = &keyO; if (old->FExpires()) { if (fRemoveExpire) { @@ -297,8 +303,13 @@ void redisDb::dbOverwriteCore(redisDb::iter itr, robj *key, robj *val, bool fUpd setMvccTstamp(val, getMvccTstamp()); } + /* Although the key is not really deleted from the database, we regard + overwrite as two steps of unlink+add, so we still need to call the unlink + callback of the module. */ + moduleNotifyKeyUnlink(key,old); + if (g_pserver->lazyfree_lazy_server_del) - freeObjAsync(itr.val()); + freeObjAsync(key, itr.val()); else decrRefCount(itr.val()); @@ -315,11 +326,11 @@ void dbOverwrite(redisDb *db, robj *key, robj *val, bool fRemoveExpire) { serverAssertWithInfo(NULL,key,itr != nullptr); lookupKeyUpdateObj(itr.val(), LOOKUP_NONE); - db->dbOverwriteCore(itr, key, val, !!g_pserver->fActiveReplica, fRemoveExpire); + db->dbOverwriteCore(itr, szFromObj(key), val, !!g_pserver->fActiveReplica, fRemoveExpire); } /* Insert a key, handling duplicate keys according to fReplace */ -int dbMerge(redisDb *db, robj *key, robj *val, int fReplace) +int dbMerge(redisDb *db, sds key, robj *val, int fReplace) { if (fReplace) { @@ -327,7 +338,7 @@ int dbMerge(redisDb *db, robj *key, robj *val, int fReplace) if (itr == nullptr) return (dbAddCore(db, key, val, false /* fUpdateMvcc */) == true); - robj *old = itr.val(); + robj_roptr old = itr.val(); if (mvccFromObj(old) <= mvccFromObj(val)) { db->dbOverwriteCore(itr, key, val, false, true); @@ -355,7 +366,7 @@ int dbMerge(redisDb *db, robj *key, robj *val, int fReplace) * in a context where there is no clear client performing the operation. */ void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal) { db->prepOverwriteForSnapshot(szFromObj(key)); - if (!dbAddCore(db, key, val, true /* fUpdateMvcc */)) { + if (!dbAddCore(db, szFromObj(key), val, true /* fUpdateMvcc */)) { dbOverwrite(db, key, val, !keepttl); } incrRefCount(val); @@ -367,12 +378,6 @@ void setKey(client *c, redisDb *db, robj *key, robj *val) { genericSetKey(c,db,key,val,0,1); } -/* Return true if the specified key exists in the specified database. - * LRU/LFU info is not updated in any way. */ -int dbExists(redisDb *db, robj *key) { - return (db->find(key) != nullptr); -} - /* Return a random key, in form of a Redis object. * If there are no keys, NULL is returned. * @@ -427,12 +432,16 @@ bool redisDbPersistentData::syncDelete(robj *key) if (itr != nullptr && itr.val()->FExpires()) removeExpire(key, itr); + robj_sharedptr val(itr.val()); bool fDeleted = false; if (m_spstorage != nullptr) fDeleted = m_spstorage->erase(szFromObj(key)); fDeleted = (dictDelete(m_pdict,ptrFromObj(key)) == DICT_OK) || fDeleted; if (fDeleted) { + /* Tells the module that the key has been unlinked from the database. */ + moduleNotifyKeyUnlink(key,val); // MODULE Compat Note: We should be giving the actual key value here + dictEntry *de = dictUnlink(m_dictChanged, szFromObj(key)); if (de != nullptr) { @@ -534,7 +543,6 @@ long long emptyDbStructure(redisDb **dbarray, int dbnum, int async, dbarray[j]->clear(async, callback); /* Because all keys of database are removed, reset average ttl. */ dbarray[j]->avg_ttl = 0; - dbarray[j]->last_expire_set = 0; } return removed; @@ -572,7 +580,7 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { /* Make sure the WATCHed keys are affected by the FLUSH* commands. * Note that we need to call the function while the keys are still * there. */ - signalFlushedDb(dbnum); + signalFlushedDb(dbnum, async); /* Empty redis database structure. */ removed = emptyDbStructure(g_pserver->db, dbnum, async, callback); @@ -613,6 +621,10 @@ const dbBackup *backupDb(void) { sizeof(g_pserver->cluster->slots_keys_count)); } + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE, + NULL); + return backup; } @@ -630,8 +642,13 @@ void discardDbBackup(const dbBackup *backup, int flags, void(callback)(void*)) { /* Release slots to keys map backup if enable cluster. */ if (g_pserver->cluster_enabled) freeSlotsToKeysMap(backup->slots_to_keys, async); + /* Release buckup. */ zfree(backup->dbarray); delete backup; + + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD, + NULL); } /* Restore the previously created backup (discarding what currently resides @@ -656,6 +673,10 @@ void restoreDbBackup(const dbBackup *backup) { /* Release buckup. */ zfree(backup->dbarray); delete backup; + + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE, + NULL); } int selectDb(client *c, int id) { @@ -690,7 +711,7 @@ void signalModifiedKey(client *c, redisDb *db, robj *key) { trackingInvalidateKey(c,key); } -void signalFlushedDb(int dbid) { +void signalFlushedDb(int dbid, int async) { int startdb, enddb; if (dbid == -1) { startdb = 0; @@ -703,7 +724,7 @@ void signalFlushedDb(int dbid) { touchAllWatchedKeysInDb(g_pserver->db[j], NULL); } - trackingInvalidateKeysOnFlush(dbid); + trackingInvalidateKeysOnFlush(async); } /*----------------------------------------------------------------------------- @@ -713,21 +734,23 @@ void signalFlushedDb(int dbid) { /* Return the set of flags to use for the emptyDb() call for FLUSHALL * and FLUSHDB commands. * - * Currently the command just attempts to parse the "ASYNC" option. It - * also checks if the command arity is wrong. + * sync: flushes the database in an sync manner. + * async: flushes the database in an async manner. + * no option: determine sync or async according to the value of lazyfree-lazy-user-flush. * * On success C_OK is returned and the flags are stored in *flags, otherwise * C_ERR is returned and the function sends an error to the client. */ int getFlushCommandFlags(client *c, int *flags) { /* Parse the optional ASYNC option. */ - if (c->argc > 1) { - if (c->argc > 2 || strcasecmp(szFromObj(c->argv[1]),"async")) { - addReply(c,shared.syntaxerr); - return C_ERR; - } - *flags = EMPTYDB_ASYNC; - } else { + if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"sync")) { *flags = EMPTYDB_NO_FLAGS; + } else if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"async")) { + *flags = EMPTYDB_ASYNC; + } else if (c->argc == 1) { + *flags = g_pserver->lazyfree_lazy_user_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; + } else { + addReplyErrorObject(c,shared.syntaxerr); + return C_ERR; } return C_OK; } @@ -745,6 +768,9 @@ void flushAllDataAndResetRDB(int flags) { rdbSave(nullptr, rsiptr); g_pserver->dirty = saved_dirty; } + + /* Without that extra dirty++, when db was already empty, FLUSHALL will + * not be replicated nor put into the AOF. */ g_pserver->dirty++; #if defined(USE_JEMALLOC) /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. @@ -862,10 +888,9 @@ void mexistsCommand(client *c) { } void selectCommand(client *c) { - long id; + int id; - if (getLongFromObjectOrReply(c, c->argv[1], &id, - "invalid DB index") != C_OK) + if (getIntFromObjectOrReply(c, c->argv[1], &id, NULL) != C_OK) return; if (g_pserver->cluster_enabled && id != 0) { @@ -1131,7 +1156,7 @@ void scanGenericCommand(client *c, robj_roptr o, unsigned long cursor) { } if (count < 1) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); goto cleanup; } @@ -1150,7 +1175,7 @@ void scanGenericCommand(client *c, robj_roptr o, unsigned long cursor) { type = szFromObj(c->argv[i+1]); i+= 2; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); goto cleanup; } } @@ -1310,7 +1335,7 @@ void scanFilterAndReply(client *c, list *keys, sds pat, sds type, int use_patter int filter = 0; /* Filter element if it does not match the pattern. */ - if (!filter && use_pattern) { + if (use_pattern) { if (filterKey(kobj, pat, patlen)) filter = 1; } @@ -1336,6 +1361,7 @@ void scanFilterAndReply(client *c, list *keys, sds pat, sds type, int use_patter * value, or skip it if it was not filtered: we only match keys. */ if (o && (o->type == OBJ_ZSET || o->type == OBJ_HASH)) { node = nextnode; + serverAssert(node); /* assertion for valgrind (avoid NPD) */ nextnode = listNextNode(node); if (filter) { kobj = (robj*)listNodeValue(node); @@ -1405,7 +1431,7 @@ void shutdownCommand(client *c) { int flags = 0; if (c->argc > 2) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } else if (c->argc == 2) { if (!strcasecmp(szFromObj(c->argv[1]),"nosave")) { @@ -1413,7 +1439,7 @@ void shutdownCommand(client *c) { } else if (!strcasecmp(szFromObj(c->argv[1]),"save")) { flags |= SHUTDOWN_SAVE; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -1483,8 +1509,7 @@ void renamenxCommand(client *c) { void moveCommand(client *c) { robj *o; redisDb *src, *dst; - int srcid; - long long dbid; + int srcid, dbid; if (g_pserver->cluster_enabled) { addReplyError(c,"MOVE is not allowed in cluster mode"); @@ -1495,11 +1520,11 @@ void moveCommand(client *c) { src = c->db; srcid = c->db->id; - if (getLongLongFromObject(c->argv[2],&dbid) == C_ERR || - dbid < INT_MIN || dbid > INT_MAX || - selectDb(c,dbid) == C_ERR) - { - addReply(c,shared.outofrangeerr); + if (getIntFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) + return; + + if (selectDb(c,dbid) == C_ERR) { + addReplyError(c,"DB index is out of range"); return; } dst = c->db; @@ -1508,7 +1533,7 @@ void moveCommand(client *c) { /* If the user is moving using as target the same * DB as the source DB it is probably an error. */ if (src == dst) { - addReply(c,shared.sameobjecterr); + addReplyErrorObject(c,shared.sameobjecterr); return; } @@ -1552,6 +1577,110 @@ void moveCommand(client *c) { addReply(c,shared.cone); } +void copyCommand(client *c) { + robj *o; + redisDb *src, *dst; + int srcid, dbid; + expireEntry *expire = nullptr; + int j, replace = 0, fdelete = 0; + + /* Obtain source and target DB pointers + * Default target DB is the same as the source DB + * Parse the REPLACE option and targetDB option. */ + src = c->db; + dst = c->db; + srcid = c->db->id; + dbid = c->db->id; + for (j = 3; j < c->argc; j++) { + int additional = c->argc - j - 1; + if (!strcasecmp(szFromObj(c->argv[j]),"replace")) { + replace = 1; + } else if (!strcasecmp(szFromObj(c->argv[j]), "db") && additional >= 1) { + if (getIntFromObjectOrReply(c, c->argv[j+1], &dbid, NULL) != C_OK) + return; + + if (selectDb(c, dbid) == C_ERR) { + addReplyError(c,"DB index is out of range"); + return; + } + dst = c->db; + selectDb(c,srcid); /* Back to the source DB */ + j++; /* Consume additional arg. */ + } else { + addReplyErrorObject(c,shared.syntaxerr); + return; + } + } + + if ((g_pserver->cluster_enabled == 1) && (srcid != 0 || dbid != 0)) { + addReplyError(c,"Copying to another database is not allowed in cluster mode"); + return; + } + + /* If the user select the same DB as + * the source DB and using newkey as the same key + * it is probably an error. */ + robj *key = c->argv[1]; + robj *newkey = c->argv[2]; + if (src == dst && (sdscmp(szFromObj(key), szFromObj(newkey)) == 0)) { + addReplyErrorObject(c,shared.sameobjecterr); + return; + } + + /* Check if the element exists and get a reference */ + o = lookupKeyWrite(c->db, key); + if (!o) { + addReply(c,shared.czero); + return; + } + expire = c->db->getExpire(key); + + /* Return zero if the key already exists in the target DB. + * If REPLACE option is selected, delete newkey from targetDB. */ + if (lookupKeyWrite(dst,newkey) != NULL) { + if (replace) { + fdelete = 1; + } else { + addReply(c,shared.czero); + return; + } + } + + /* Duplicate object according to object's type. */ + robj *newobj; + switch(o->type) { + case OBJ_STRING: newobj = dupStringObject(o); break; + case OBJ_LIST: newobj = listTypeDup(o); break; + case OBJ_SET: newobj = setTypeDup(o); break; + case OBJ_ZSET: newobj = zsetDup(o); break; + case OBJ_HASH: newobj = hashTypeDup(o); break; + case OBJ_STREAM: newobj = streamDup(o); break; + case OBJ_MODULE: + newobj = moduleTypeDupOrReply(c, key, newkey, o); + if (!newobj) return; + break; + default: + addReplyError(c, "unknown type object"); + return; + } + + if (fdelete) { + dbDelete(dst,newkey); + } + + dbAdd(dst,newkey,newobj); + if (expire != nullptr) { + if (expire != nullptr) setExpire(c, dst, newkey, expire->duplicate()); + } + + /* OK! key copied */ + signalModifiedKey(c,dst,c->argv[2]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"copy_to",c->argv[2],dst->id); + + g_pserver->dirty++; + addReply(c,shared.cone); +} + /* Helper function for dbSwapDatabases(): scans the list of keys that have * one or more blocked clients for B[LR]POP or other blocking commands * and signal the keys as ready if they are of the right type. See the comment @@ -1562,10 +1691,7 @@ void scanDatabaseForReadyLists(redisDb *db) { while((de = dictNext(di)) != NULL) { robj *key = (robj*)dictGetKey(de); robj *value = lookupKey(db,key,LOOKUP_NOTOUCH); - if (value && (value->type == OBJ_LIST || - value->type == OBJ_STREAM || - value->type == OBJ_ZSET)) - signalKeyAsReady(db, key); + if (value) signalKeyAsReady(db, key, value->type); } dictReleaseIterator(di); } @@ -1578,7 +1704,7 @@ void scanDatabaseForReadyLists(redisDb *db) { * * Returns C_ERR if at least one of the DB ids are out of range, otherwise * C_OK is returned. */ -int dbSwapDatabases(long id1, long id2) { +int dbSwapDatabases(int id1, int id2) { if (id1 < 0 || id1 >= cserver.dbnum || id2 < 0 || id2 >= cserver.dbnum) return C_ERR; if (id1 == id2) return C_OK; @@ -1612,7 +1738,7 @@ int dbSwapDatabases(long id1, long id2) { /* SWAPDB db1 db2 */ void swapdbCommand(client *c) { - long id1, id2; + int id1, id2; /* Not allowed in cluster mode: we have just DB 0 there. */ if (g_pserver->cluster_enabled) { @@ -1621,11 +1747,11 @@ void swapdbCommand(client *c) { } /* Get the two DBs indexes. */ - if (getLongFromObjectOrReply(c, c->argv[1], &id1, + if (getIntFromObjectOrReply(c, c->argv[1], &id1, "invalid first DB index") != C_OK) return; - if (getLongFromObjectOrReply(c, c->argv[2], &id2, + if (getIntFromObjectOrReply(c, c->argv[2], &id2, "invalid second DB index") != C_OK) return; @@ -1809,11 +1935,13 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { incrRefCount(argv[0]); incrRefCount(argv[1]); - if (g_pserver->aof_state != AOF_OFF) - feedAppendOnlyFile(cserver.delCommand,db->id,argv,2); - // Active replicas do their own expiries, do not propogate - if (!g_pserver->fActiveReplica) - replicationFeedSlaves(g_pserver->slaves,db->id,argv,2); + /* If the master decided to expire a key we must propagate it to replicas no matter what.. + * Even if module executed a command without asking for propagation. */ + int prev_replication_allowed = g_pserver->replication_allowed; + g_pserver->replication_allowed = 1; + if (!g_pserver->fActiveReplica) // Active replicas do their own expiries, do not propogate + propagate(cserver.delCommand,db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + g_pserver->replication_allowed = prev_replication_allowed; decrRefCount(argv[0]); decrRefCount(argv[1]); @@ -1944,6 +2072,12 @@ int expireIfNeeded(redisDb *db, robj *key) { * we think the key is expired at this time. */ if (listLength(g_pserver->masters) && !g_pserver->fActiveReplica) return 1; + /* If clients are paused, we keep the current dataset constant, + * but return to the client what we believe is the right state. Typically, + * at the end of the pause we will properly expire the key OR we will + * have failed over and the new primary will send us the expire. */ + if (checkClientPauseTimeoutAndReturnIfPaused()) return 1; + /* Delete the key */ g_pserver->stat_expiredkeys++; propagateExpire(db,key,g_pserver->lazyfree_lazy_expire); @@ -2020,7 +2154,7 @@ int getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, get result->numkeys = 0; return 0; } else { - serverPanic("Redis built-in command declared keys positions not matching the arity requirements."); + serverPanic("KeyDB built-in command declared keys positions not matching the arity requirements."); } } keys[i++] = j; @@ -2057,58 +2191,53 @@ void getKeysFreeResult(getKeysResult *result) { } /* Helper function to extract keys from following commands: + * COMMAND [destkey] [...] [...] ... + * + * eg: + * ZUNION ... * ZUNIONSTORE ... - * ZINTERSTORE ... */ -int zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { + * + * 'storeKeyOfs': destkey index, 0 means destkey not exists. + * 'keyCountOfs': num-keys index. + * 'firstKeyOfs': firstkey index. + * 'keyStep': the interval of each key, usually this value is 1. + * */ +int genericGetKeys(int storeKeyOfs, int keyCountOfs, int firstKeyOfs, int keyStep, + robj **argv, int argc, getKeysResult *result) { int i, num, *keys; - UNUSED(cmd); - num = atoi(szFromObj(argv[2])); + num = atoi(szFromObj(argv[keyCountOfs])); /* Sanity check. Don't return any key if the command is going to - * reply with syntax error. */ - if (num < 1 || num > (argc-3)) { + * reply with syntax error. (no input keys). */ + if (num < 1 || num > (argc - firstKeyOfs)/keyStep) { result->numkeys = 0; return 0; } - /* Keys in z{union,inter}store come from two places: - * argv[1] = storage key, - * argv[3...n] = keys to intersect */ - /* Total keys = {union,inter} keys + storage key */ - keys = getKeysPrepareResult(result, num+1); - result->numkeys = num+1; + int numkeys = storeKeyOfs ? num + 1 : num; + keys = getKeysPrepareResult(result, numkeys); + result->numkeys = numkeys; - /* Add all key positions for argv[3...n] to keys[] */ - for (i = 0; i < num; i++) keys[i] = 3+i; - - /* Finally add the argv[1] key position (the storage key target). */ - keys[num] = 1; + /* Add all key positions for argv[firstKeyOfs...n] to keys[] */ + for (i = 0; i < num; i++) keys[i] = firstKeyOfs+(i*keyStep); + if (storeKeyOfs) keys[num] = storeKeyOfs; return result->numkeys; } -/* Helper function to extract keys from the following commands: - * EVAL