
* add docker build * fix the working dir in Dockerfile * add release publish docker image * address intentation and use default release * migrate keydb_modstatsd to keydb-internal * rm * add submodule cpp-statsd-client * include trigger keydb_modstatsd Makefile in modules Makefile * update * have clean also trigger keydb_modstatsd clean * move cpp-statsd-client to deps * checkout to a06a5b9359f31d946fe163b9038586982971ae49 * update relative path in compilation * remove submodule instead use the source * include building keydb statsd module * fix check in Dockerfile docker-entrypoint.sh * fix * fix the comment caused stuck docker build * use commit hash as tag template * fix * test tag * Revert "test tag" This reverts commit 9cbc57137d57aab4fdd5a9283bae07391b3c7f8b. * make docker build independent * add new build to ci * emit system free metrics with '/proc/meminfo' * have emit system free memory within metrics_time_taken_us and also add metric time taken for it * Remove Expireset (#217) Major refactor to place expiry information directly in the object struct. * update MemFree to MemAvailable in keydb statsd * add metric emit for non-empty primary with less than 2 connected replicas * address comments * Multiply CPU percent metric by 100 * Fix memory leaks * Fix slow to free when low lock contention * fix nodename metricsname * fix unnecessary replace * Make propagating before freeing module context optional (#225) * don't propogate on module context free for rdb load * default in wrong place * Flash expiration (#197) Design Doc: https://docs.google.com/document/d/1NmnYGnHLdZp-KOUCUatX5iXpF-L3YK4VUc9Lm3Tqxpo/edit?usp=sharing * Emit more aggregate metrics in modstatsd (#223) * Permit keys of differing slots as long as they are served by this cluster and we are not migrating * Fix over pessimistic checks that prevent replicas from serving mget * Fix logic bug * async rehash is preventing rehashing during RDB load after a db flush. Ensure it can't interefere after a flush * make async rehash configurable * only use internal locks when multithreaded (#205) * Fix crossslot error migrating batches of keys * Fix bug where we erroneously answer queries belonging to another shard * fix mac compile * enable level_compaction_dynamic_level_bytes after flush, and flush expires for FLASH (#229) * enable level_compaction_dynamic_level_bytes after flush, and flush expires * update debug reload for flash * update debug reload for flash complete * missing forward declare * commit existing changes then track changes for debug reload * missing args * commitChanges is conditional Co-authored-by: John Sully <jsully@snapchat.com> --------- Co-authored-by: zliang <zliang@snapchat.com> Co-authored-by: John Sully <jsully@snapchat.com> Co-authored-by: Alex Cope <acope@snapchat.com> Co-authored-by: John Sully <john@csquare.ca>
115 lines
5.8 KiB
Docker
115 lines
5.8 KiB
Docker
FROM ubuntu:20.04
|
|
SHELL ["/bin/bash","-c"]
|
|
RUN groupadd -r keydb && useradd -r -g keydb keydb
|
|
# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases
|
|
ENV GOSU_VERSION 1.14
|
|
RUN set -eux; \
|
|
savedAptMark="$(apt-mark showmanual)"; \
|
|
apt-get update; \
|
|
apt-get -o Dpkg::Options::="--force-confnew" install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \
|
|
rm -rf /var/lib/apt/lists/*; \
|
|
dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \
|
|
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \
|
|
wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \
|
|
export GNUPGHOME="$(mktemp -d)"; \
|
|
gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
|
|
gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
|
|
gpgconf --kill all; \
|
|
rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
|
|
apt-mark auto '.*' > /dev/null; \
|
|
[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
|
|
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
|
|
chmod +x /usr/local/bin/gosu; \
|
|
gosu --version; \
|
|
gosu nobody true
|
|
# build KeyDB
|
|
ARG MAKE_JOBS=""
|
|
ARG ENABLE_FLASH=""
|
|
COPY . /tmp/keydb-internal
|
|
RUN set -eux; \
|
|
cd /tmp/keydb-internal; \
|
|
savedAptMark="$(apt-mark showmanual)"; \
|
|
apt-get update; \
|
|
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confnew" install -qqy --no-install-recommends \
|
|
dpkg-dev \
|
|
pkg-config \
|
|
ca-certificates \
|
|
build-essential \
|
|
nasm \
|
|
autotools-dev \
|
|
autoconf \
|
|
libjemalloc-dev \
|
|
tcl \
|
|
tcl-dev \
|
|
uuid-dev \
|
|
libcurl4-openssl-dev \
|
|
libbz2-dev \
|
|
libzstd-dev \
|
|
liblz4-dev \
|
|
libsnappy-dev \
|
|
libssl-dev \
|
|
git; \
|
|
# disable protected mode as it relates to docker
|
|
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \
|
|
sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \
|
|
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \
|
|
make distclean; \
|
|
make -j$([ -z "$MAKE_JOBS" ] && nproc || echo "$MAKE_JOBS") BUILD_TLS=yes NO_LICENSE_CHECK=yes $([ -z "$ENABLE_FLASH" ] && echo "" || echo "ENABLE_FLASH=$ENABLE_FLASH"); \
|
|
cd src; \
|
|
mv modules/keydb_modstatsd/modstatsd.so /usr/local/lib/; \
|
|
strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \
|
|
mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \
|
|
# clean up unused dependencies
|
|
echo $savedAptMark; \
|
|
apt-mark auto '.*' > /dev/null; \
|
|
[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
|
|
find /usr/local -type f -executable -exec ldd '{}' ';' \
|
|
| awk '/=>/ { print $(NF-1) }' \
|
|
| sed 's:.*/::' \
|
|
| sort -u \
|
|
| xargs -r dpkg-query --search \
|
|
| cut -d: -f1 \
|
|
| sort -u \
|
|
| xargs -r apt-mark manual \
|
|
; \
|
|
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
|
|
rm -rf /var/lib/apt/lists/*; \
|
|
# create working directories and organize files
|
|
RUN \
|
|
mkdir /data && chown keydb:keydb /data; \
|
|
mkdir /flash && chown keydb:keydb /flash; \
|
|
mkdir -p /etc/keydb; \
|
|
cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \
|
|
sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
|
sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \
|
|
sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
|
sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \
|
|
sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
|
echo -e "\nloadmodule /usr/local/lib/modstatsd.so" >> /etc/keydb/keydb.conf; \
|
|
ln -s keydb-cli redis-cli; \
|
|
cd /etc/keydb; \
|
|
ln -s keydb.conf redis.conf; \
|
|
rm -rf /tmp/*
|
|
# generate entrypoint script
|
|
RUN set -eux; \
|
|
echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \
|
|
echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo "# perpend 'keydb-server' if not provided as first argument" >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo 'if [ "${1}" != "keydb-server" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
|
chmod +x /usr/local/bin/docker-entrypoint.sh
|
|
# set remaining image properties
|
|
VOLUME /data
|
|
WORKDIR /data
|
|
ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/
|
|
ENTRYPOINT ["docker-entrypoint.sh"]
|
|
EXPOSE 6379
|
|
CMD ["keydb-server","/etc/keydb/keydb.conf"]
|