From 9274d83feda16744f57ed65c66fc98323bd4ec33 Mon Sep 17 00:00:00 2001 From: benschermel Date: Mon, 14 Feb 2022 16:48:32 -0500 Subject: [PATCH] add Dockerfile to build from local repo --- docker-internal/Dockerfile | 10 +- docker-internal/GHE_PersonalAccessToken.txt | 2 - docker-internal/README.md | 16 ++- docker-internal/docker_PAT_build/Dockerfile | 117 ++++++++++++++++++++ 4 files changed, 136 insertions(+), 9 deletions(-) delete mode 100644 docker-internal/GHE_PersonalAccessToken.txt create mode 100644 docker-internal/docker_PAT_build/Dockerfile diff --git a/docker-internal/Dockerfile b/docker-internal/Dockerfile index 00908d41d..4e38be156 100644 --- a/docker-internal/Dockerfile +++ b/docker-internal/Dockerfile @@ -23,9 +23,9 @@ RUN set -eux; \ gosu --version; \ gosu nobody true # build KeyDB -ARG BRANCH -RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ - set -eux; \ +ARG KEYDB_DIR +COPY $KEYDB_DIR /tmp/keydb-internal/ +RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ @@ -46,9 +46,7 @@ RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ libzstd-dev \ liblz4-dev \ libsnappy-dev \ - libssl-dev \ - git; \ - cd /tmp && git clone --branch $BRANCH https://$PAT_ID:$TOKEN@github.sc-corp.net/Snapchat/keydb-internal.git --recursive; \ + libssl-dev; \ cd /tmp/keydb-internal; \ # disable protected mode as it relates to docker grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ diff --git a/docker-internal/GHE_PersonalAccessToken.txt b/docker-internal/GHE_PersonalAccessToken.txt deleted file mode 100644 index 4266d9ddf..000000000 --- a/docker-internal/GHE_PersonalAccessToken.txt +++ /dev/null @@ -1,2 +0,0 @@ -PAT_ID= -TOKEN= diff --git a/docker-internal/README.md b/docker-internal/README.md index 44ffc5538..8f5ff5c59 100644 --- a/docker-internal/README.md +++ b/docker-internal/README.md @@ -7,7 +7,20 @@ This docker image builds KeyDB within the image and cleans up afterwards. A few * keydb.conf added and linked to redis.conf for legacy compatibility and as default config file * use entrypoint and cmd for best practices. Remove protected-mode during build incase user specifies binary without .conf, or just wants append parameters -## Building the Docker Image +## Building the Docker Image Using Local KeyDB Directory + +If you have a local keydb-internal repository you would like to generate the binaries from, use the command below. This will simply copy over all the files within the local keydb-internal repo and then build the image. + +Modify the DIR build argument to your local KeyDB repo and update your image tag in the line below + +``` +docker build --build-arg DIR=/path/to/keydb-internal -t myImageName:imageTag . +``` + +Please note that directories are relative to the docker build context. You can use the `-f /path/to/Dockerfile` to specify Dockerfile which will also set the build context, your repo location will be relative to it. + + +## Building the Docker Image Using PAT & Clone This image clones the keydb-internal repo, hence a GHE PAT token or SSH access is needed. See more on [obtaining GHE PAT](https://wiki.sc-corp.net/display/TOOL/Using+the+GHE+API#UsingtheGHEAPI-Step1:PersonalTokens). It is not secure to pass tokens/credentials as build-args, env variables, or COPYing then deleting, so we use secrets. This option is only available with the [Docker BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information)so the docker build kit must be enabled via `DOCKER_BUILDKIT=1`, or permanently by appending `"features": { "buildkit": true }` to /etc/docker/daemon.json. @@ -19,6 +32,7 @@ Run the command below updating your info as follows: * Add your PAT info to GHE_PersonalAccessToken.txt ``` +cd docker_PAT_build DOCKER_BUILDKIT=1 docker build --no-cache --build-arg BRANCH=keydbpro --secret id=PAT,src=GHE_PersonalAccessToken.txt . -t myImageName:imageTag ``` diff --git a/docker-internal/docker_PAT_build/Dockerfile b/docker-internal/docker_PAT_build/Dockerfile new file mode 100644 index 000000000..00908d41d --- /dev/null +++ b/docker-internal/docker_PAT_build/Dockerfile @@ -0,0 +1,117 @@ +FROM ubuntu:18.04 +SHELL ["/bin/bash","-c"] +RUN groupadd -r keydb && useradd -r -g keydb keydb +# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true +# build KeyDB +ARG BRANCH +RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ + set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \ + dpkg-dev \ + pkg-config \ + ca-certificates \ + build-essential \ + nasm \ + autotools-dev \ + autoconf \ + libjemalloc-dev \ + tcl \ + tcl-dev \ + uuid-dev \ + libcurl4-openssl-dev \ + libbz2-dev \ + libzstd-dev \ + liblz4-dev \ + libsnappy-dev \ + libssl-dev \ + git; \ + cd /tmp && git clone --branch $BRANCH https://$PAT_ID:$TOKEN@github.sc-corp.net/Snapchat/keydb-internal.git --recursive; \ + cd /tmp/keydb-internal; \ + # disable protected mode as it relates to docker + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ + make -j$(nproc) BUILD_TLS=yes NO_LICENSE_CHECK=yes; \ + cd src; \ + strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ + mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ + # clean up unused dependencies + echo $savedAptMark; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + ; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ +# create working directories and organize files +RUN \ + mkdir /data && chown keydb:keydb /data; \ + mkdir /flash && chown keydb:keydb /flash; \ + mkdir -p /etc/keydb; \ + cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \ + sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ + sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ + ln -s keydb-cli redis-cli; \ + cd /etc/keydb; \ + ln -s keydb.conf redis.conf; \ + rm -rf /tmp/* +# generate entrypoint script +RUN set -eux; \ + echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ + echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + chmod +x /usr/local/bin/docker-entrypoint.sh +# set remaining image properties +VOLUME /data +WORKDIR /data +ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] +EXPOSE 6379 +CMD ["keydb-server","/etc/keydb/keydb.conf"] + + + + +