From eb7ca372d4c12fdc6951036090e6478588bf7a35 Mon Sep 17 00:00:00 2001 From: benschermel Date: Mon, 14 Feb 2022 10:12:03 -0500 Subject: [PATCH 1/7] add Dockerfile for KeyDB internal image --- docker-internal/Dockerfile | 117 ++++++++++++++++++++ docker-internal/GHE_PersonalAccessToken.txt | 2 + docker-internal/README.md | 45 ++++++++ 3 files changed, 164 insertions(+) create mode 100644 docker-internal/Dockerfile create mode 100644 docker-internal/GHE_PersonalAccessToken.txt create mode 100644 docker-internal/README.md diff --git a/docker-internal/Dockerfile b/docker-internal/Dockerfile new file mode 100644 index 000000000..00908d41d --- /dev/null +++ b/docker-internal/Dockerfile @@ -0,0 +1,117 @@ +FROM ubuntu:18.04 +SHELL ["/bin/bash","-c"] +RUN groupadd -r keydb && useradd -r -g keydb keydb +# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true +# build KeyDB +ARG BRANCH +RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ + set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \ + dpkg-dev \ + pkg-config \ + ca-certificates \ + build-essential \ + nasm \ + autotools-dev \ + autoconf \ + libjemalloc-dev \ + tcl \ + tcl-dev \ + uuid-dev \ + libcurl4-openssl-dev \ + libbz2-dev \ + libzstd-dev \ + liblz4-dev \ + libsnappy-dev \ + libssl-dev \ + git; \ + cd /tmp && git clone --branch $BRANCH https://$PAT_ID:$TOKEN@github.sc-corp.net/Snapchat/keydb-internal.git --recursive; \ + cd /tmp/keydb-internal; \ + # disable protected mode as it relates to docker + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ + make -j$(nproc) BUILD_TLS=yes NO_LICENSE_CHECK=yes; \ + cd src; \ + strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ + mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ + # clean up unused dependencies + echo $savedAptMark; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + ; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ +# create working directories and organize files +RUN \ + mkdir /data && chown keydb:keydb /data; \ + mkdir /flash && chown keydb:keydb /flash; \ + mkdir -p /etc/keydb; \ + cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \ + sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ + sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ + ln -s keydb-cli redis-cli; \ + cd /etc/keydb; \ + ln -s keydb.conf redis.conf; \ + rm -rf /tmp/* +# generate entrypoint script +RUN set -eux; \ + echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ + echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + chmod +x /usr/local/bin/docker-entrypoint.sh +# set remaining image properties +VOLUME /data +WORKDIR /data +ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] +EXPOSE 6379 +CMD ["keydb-server","/etc/keydb/keydb.conf"] + + + + + diff --git a/docker-internal/GHE_PersonalAccessToken.txt b/docker-internal/GHE_PersonalAccessToken.txt new file mode 100644 index 000000000..4266d9ddf --- /dev/null +++ b/docker-internal/GHE_PersonalAccessToken.txt @@ -0,0 +1,2 @@ +PAT_ID= +TOKEN= diff --git a/docker-internal/README.md b/docker-internal/README.md new file mode 100644 index 000000000..3a5769398 --- /dev/null +++ b/docker-internal/README.md @@ -0,0 +1,45 @@ +# KeyDB Docker Image (Snap Internal) + +This docker image builds KeyDB within the image and cleans up afterwards. A few notes about the image: +* use gosu to avoid running as root https://github.com/tianon/gosu +* packages are installed, tracked and cleaned up to minimize container sizes +* keydb-server binary symbols remain for troubleshooting. All other KeyDB binaries are stripped +* keydb.conf added and linked to redis.conf for legacy compatibility and as default config file +* use entrypoint and cmd for best practices. Remove protected-mode during build incase user specifies binary without .conf, or just wants append parameters + +## Building the Docker Image + +This image clones the keydb-internal repo, hence a GHE PAT token or SSH access is needed. See more on [obtaining GHE PAT](https://wiki.sc-corp.net/display/TOOL/Using+the+GHE+API#UsingtheGHEAPI-Step1:PersonalTokens). It is not secure to pass tokens/credentials as build-args, env variables, or COPYing then deleting, so we use secrets. This option is only available with the [Docker BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information)so the docker build kit must be enabled via `DOCKER_BUILDKIT=1`, or permanently by appending `"features": { "buildkit": true }` to /etc/docker/daemon.json. + +#### To Build: + +Run the command below updating your info as follows: +* Pass in the branch name as a build argument +* Tag your image to whatever you want to use +* Add your PAT info to GHE_PersonalAccessToken.txt + +``` +DOCKER_BUILDKIT=1 docker build --no-cache --build-arg BRANCH=keydbpro --secret id=PAT,src=GHE_PersonalAccessToken.txt . -t myImageName:imageTag +``` + +For a detailed conventional build view pass `--progress=plain` while building. Note that `--no-cache` must be used to ensure your credentials are pulled in as they are not cached. + +## Using The Image + +#### Bind port +Bind the container to the node/machine by passing the parameter `-p 6379:6379` when you run the docker container + +#### Configuration file +By default KeyDB launches by specifiying its internal keydb.conf file at `/etc/keydb/keydb.conf`. If you would like to use your own config file you can link to the config file with `-v /path-to-config-file/keydb.conf:/etc/keydb/keydb.conf` then run the container whose default command is `keydb-server /etc/keydb/keydb.conf` + +Alternatively specify any config file location, by specifying the location following the binary to override default `docker run ThisDockerImage keydb-server /path/to/my/config-file/`. You can also just append parameters to default image with `docker run ThisDockerImage keydb-server /etc/keydb/keydb.conf --dir /data --server-threads 4`. + +#### Persistent storage +If persistence is enabled, data is stored in the VOLUME /data, which can be used with --volumes-from some-volume-container or -v /docker/host/dir:/data (see docs.docker volumes). + +#### Connect via keydb-cli +you can grab the ip of the container with `docker inspect --format '{{ .NetworkSettings.IPAddress }}' mycontainername` then run the following: +``` +docker run -it --rm ThisDockerImage keydb-cli -h -p 6379 +``` + From 505d76196b1a6c64d54ecc04e6c20ad1c201d69a Mon Sep 17 00:00:00 2001 From: benschermel Date: Mon, 14 Feb 2022 10:17:33 -0500 Subject: [PATCH 2/7] add .gitignore for PAT --- docker-internal/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 docker-internal/.gitignore diff --git a/docker-internal/.gitignore b/docker-internal/.gitignore new file mode 100644 index 000000000..cbee8387c --- /dev/null +++ b/docker-internal/.gitignore @@ -0,0 +1 @@ +GHE_PersonalAccessToken.txt From 4f596274f818c8dda5fd436d127e9dd5a256daa7 Mon Sep 17 00:00:00 2001 From: benschermel Date: Mon, 14 Feb 2022 11:53:03 -0500 Subject: [PATCH 3/7] update readme for mac build --- docker-internal/README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docker-internal/README.md b/docker-internal/README.md index 3a5769398..44ffc5538 100644 --- a/docker-internal/README.md +++ b/docker-internal/README.md @@ -24,21 +24,34 @@ DOCKER_BUILDKIT=1 docker build --no-cache --build-arg BRANCH=keydbpro --secret i For a detailed conventional build view pass `--progress=plain` while building. Note that `--no-cache` must be used to ensure your credentials are pulled in as they are not cached. +#### Troubleshooting + +If you are building on macOS or with docker frontend you may need to append the following line to the top of the Dockerfile: + +``` +# syntax=docker/dockerfile:1.3 +``` + ## Using The Image #### Bind port + Bind the container to the node/machine by passing the parameter `-p 6379:6379` when you run the docker container #### Configuration file + By default KeyDB launches by specifiying its internal keydb.conf file at `/etc/keydb/keydb.conf`. If you would like to use your own config file you can link to the config file with `-v /path-to-config-file/keydb.conf:/etc/keydb/keydb.conf` then run the container whose default command is `keydb-server /etc/keydb/keydb.conf` -Alternatively specify any config file location, by specifying the location following the binary to override default `docker run ThisDockerImage keydb-server /path/to/my/config-file/`. You can also just append parameters to default image with `docker run ThisDockerImage keydb-server /etc/keydb/keydb.conf --dir /data --server-threads 4`. +Alternatively specify any config file location, by specifying the location following the binary to override default: `docker run ThisDockerImage keydb-server /path/to/my/config-file/`. You can also just append parameters to default image with `docker run ThisDockerImage keydb-server /etc/keydb/keydb.conf --dir /data --server-threads 4`. #### Persistent storage -If persistence is enabled, data is stored in the VOLUME /data, which can be used with --volumes-from some-volume-container or -v /docker/host/dir:/data (see docs.docker volumes). + +If persistence is enabled, data is stored in the VOLUME /data, which can be used with --volumes-from some-volume-container or -v /docker/host/dir:/data (see [docs.docker volumes](https://docs.docker.com/storage/volumes/)). #### Connect via keydb-cli + you can grab the ip of the container with `docker inspect --format '{{ .NetworkSettings.IPAddress }}' mycontainername` then run the following: + ``` docker run -it --rm ThisDockerImage keydb-cli -h -p 6379 ``` From 9274d83feda16744f57ed65c66fc98323bd4ec33 Mon Sep 17 00:00:00 2001 From: benschermel Date: Mon, 14 Feb 2022 16:48:32 -0500 Subject: [PATCH 4/7] add Dockerfile to build from local repo --- docker-internal/Dockerfile | 10 +- docker-internal/GHE_PersonalAccessToken.txt | 2 - docker-internal/README.md | 16 ++- docker-internal/docker_PAT_build/Dockerfile | 117 ++++++++++++++++++++ 4 files changed, 136 insertions(+), 9 deletions(-) delete mode 100644 docker-internal/GHE_PersonalAccessToken.txt create mode 100644 docker-internal/docker_PAT_build/Dockerfile diff --git a/docker-internal/Dockerfile b/docker-internal/Dockerfile index 00908d41d..4e38be156 100644 --- a/docker-internal/Dockerfile +++ b/docker-internal/Dockerfile @@ -23,9 +23,9 @@ RUN set -eux; \ gosu --version; \ gosu nobody true # build KeyDB -ARG BRANCH -RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ - set -eux; \ +ARG KEYDB_DIR +COPY $KEYDB_DIR /tmp/keydb-internal/ +RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ @@ -46,9 +46,7 @@ RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ libzstd-dev \ liblz4-dev \ libsnappy-dev \ - libssl-dev \ - git; \ - cd /tmp && git clone --branch $BRANCH https://$PAT_ID:$TOKEN@github.sc-corp.net/Snapchat/keydb-internal.git --recursive; \ + libssl-dev; \ cd /tmp/keydb-internal; \ # disable protected mode as it relates to docker grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ diff --git a/docker-internal/GHE_PersonalAccessToken.txt b/docker-internal/GHE_PersonalAccessToken.txt deleted file mode 100644 index 4266d9ddf..000000000 --- a/docker-internal/GHE_PersonalAccessToken.txt +++ /dev/null @@ -1,2 +0,0 @@ -PAT_ID= -TOKEN= diff --git a/docker-internal/README.md b/docker-internal/README.md index 44ffc5538..8f5ff5c59 100644 --- a/docker-internal/README.md +++ b/docker-internal/README.md @@ -7,7 +7,20 @@ This docker image builds KeyDB within the image and cleans up afterwards. A few * keydb.conf added and linked to redis.conf for legacy compatibility and as default config file * use entrypoint and cmd for best practices. Remove protected-mode during build incase user specifies binary without .conf, or just wants append parameters -## Building the Docker Image +## Building the Docker Image Using Local KeyDB Directory + +If you have a local keydb-internal repository you would like to generate the binaries from, use the command below. This will simply copy over all the files within the local keydb-internal repo and then build the image. + +Modify the DIR build argument to your local KeyDB repo and update your image tag in the line below + +``` +docker build --build-arg DIR=/path/to/keydb-internal -t myImageName:imageTag . +``` + +Please note that directories are relative to the docker build context. You can use the `-f /path/to/Dockerfile` to specify Dockerfile which will also set the build context, your repo location will be relative to it. + + +## Building the Docker Image Using PAT & Clone This image clones the keydb-internal repo, hence a GHE PAT token or SSH access is needed. See more on [obtaining GHE PAT](https://wiki.sc-corp.net/display/TOOL/Using+the+GHE+API#UsingtheGHEAPI-Step1:PersonalTokens). It is not secure to pass tokens/credentials as build-args, env variables, or COPYing then deleting, so we use secrets. This option is only available with the [Docker BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information)so the docker build kit must be enabled via `DOCKER_BUILDKIT=1`, or permanently by appending `"features": { "buildkit": true }` to /etc/docker/daemon.json. @@ -19,6 +32,7 @@ Run the command below updating your info as follows: * Add your PAT info to GHE_PersonalAccessToken.txt ``` +cd docker_PAT_build DOCKER_BUILDKIT=1 docker build --no-cache --build-arg BRANCH=keydbpro --secret id=PAT,src=GHE_PersonalAccessToken.txt . -t myImageName:imageTag ``` diff --git a/docker-internal/docker_PAT_build/Dockerfile b/docker-internal/docker_PAT_build/Dockerfile new file mode 100644 index 000000000..00908d41d --- /dev/null +++ b/docker-internal/docker_PAT_build/Dockerfile @@ -0,0 +1,117 @@ +FROM ubuntu:18.04 +SHELL ["/bin/bash","-c"] +RUN groupadd -r keydb && useradd -r -g keydb keydb +# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true +# build KeyDB +ARG BRANCH +RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ + set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \ + dpkg-dev \ + pkg-config \ + ca-certificates \ + build-essential \ + nasm \ + autotools-dev \ + autoconf \ + libjemalloc-dev \ + tcl \ + tcl-dev \ + uuid-dev \ + libcurl4-openssl-dev \ + libbz2-dev \ + libzstd-dev \ + liblz4-dev \ + libsnappy-dev \ + libssl-dev \ + git; \ + cd /tmp && git clone --branch $BRANCH https://$PAT_ID:$TOKEN@github.sc-corp.net/Snapchat/keydb-internal.git --recursive; \ + cd /tmp/keydb-internal; \ + # disable protected mode as it relates to docker + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ + make -j$(nproc) BUILD_TLS=yes NO_LICENSE_CHECK=yes; \ + cd src; \ + strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ + mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ + # clean up unused dependencies + echo $savedAptMark; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + ; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ +# create working directories and organize files +RUN \ + mkdir /data && chown keydb:keydb /data; \ + mkdir /flash && chown keydb:keydb /flash; \ + mkdir -p /etc/keydb; \ + cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \ + sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ + sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ + ln -s keydb-cli redis-cli; \ + cd /etc/keydb; \ + ln -s keydb.conf redis.conf; \ + rm -rf /tmp/* +# generate entrypoint script +RUN set -eux; \ + echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ + echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + chmod +x /usr/local/bin/docker-entrypoint.sh +# set remaining image properties +VOLUME /data +WORKDIR /data +ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] +EXPOSE 6379 +CMD ["keydb-server","/etc/keydb/keydb.conf"] + + + + + From 0517daa3084c1e8163788f06e0c7ec576dee12bc Mon Sep 17 00:00:00 2001 From: benschermel Date: Mon, 14 Feb 2022 17:33:05 -0500 Subject: [PATCH 5/7] fix for 20.04 build --- docker-internal/.gitignore | 1 - docker-internal/Dockerfile | 3 ++- docker-internal/docker_PAT_build/Dockerfile | 3 ++- docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt | 2 ++ 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt diff --git a/docker-internal/.gitignore b/docker-internal/.gitignore index cbee8387c..e69de29bb 100644 --- a/docker-internal/.gitignore +++ b/docker-internal/.gitignore @@ -1 +0,0 @@ -GHE_PersonalAccessToken.txt diff --git a/docker-internal/Dockerfile b/docker-internal/Dockerfile index 4e38be156..9a849d276 100644 --- a/docker-internal/Dockerfile +++ b/docker-internal/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 SHELL ["/bin/bash","-c"] RUN groupadd -r keydb && useradd -r -g keydb keydb # use gosu for easy step-down from root: https://github.com/tianon/gosu/releases @@ -62,6 +62,7 @@ RUN set -eux; \ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ find /usr/local -type f -executable -exec ldd '{}' ';' \ | awk '/=>/ { print $(NF-1) }' \ + | sed 's:.*/::' \ | sort -u \ | xargs -r dpkg-query --search \ | cut -d: -f1 \ diff --git a/docker-internal/docker_PAT_build/Dockerfile b/docker-internal/docker_PAT_build/Dockerfile index 00908d41d..e27c4388a 100644 --- a/docker-internal/docker_PAT_build/Dockerfile +++ b/docker-internal/docker_PAT_build/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 SHELL ["/bin/bash","-c"] RUN groupadd -r keydb && useradd -r -g keydb keydb # use gosu for easy step-down from root: https://github.com/tianon/gosu/releases @@ -64,6 +64,7 @@ RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ find /usr/local -type f -executable -exec ldd '{}' ';' \ | awk '/=>/ { print $(NF-1) }' \ + | sed 's:.*/::' \ | sort -u \ | xargs -r dpkg-query --search \ | cut -d: -f1 \ diff --git a/docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt b/docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt new file mode 100644 index 000000000..4266d9ddf --- /dev/null +++ b/docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt @@ -0,0 +1,2 @@ +PAT_ID= +TOKEN= From f61a0cf31d8ec8f0e9e7bc3662e12698cbb4ede1 Mon Sep 17 00:00:00 2001 From: benschermel Date: Mon, 14 Feb 2022 17:34:03 -0500 Subject: [PATCH 6/7] update gitignore --- docker-internal/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-internal/.gitignore b/docker-internal/.gitignore index e69de29bb..cbee8387c 100644 --- a/docker-internal/.gitignore +++ b/docker-internal/.gitignore @@ -0,0 +1 @@ +GHE_PersonalAccessToken.txt From 105359157b508e0cbde2c6395175d079aa6d7bf8 Mon Sep 17 00:00:00 2001 From: benschermel Date: Tue, 15 Feb 2022 00:56:49 -0500 Subject: [PATCH 7/7] update build command to squash layers on COPY --- docker-internal/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-internal/README.md b/docker-internal/README.md index 8f5ff5c59..8a44bc7ba 100644 --- a/docker-internal/README.md +++ b/docker-internal/README.md @@ -11,10 +11,12 @@ This docker image builds KeyDB within the image and cleans up afterwards. A few If you have a local keydb-internal repository you would like to generate the binaries from, use the command below. This will simply copy over all the files within the local keydb-internal repo and then build the image. +One issue with COPY is that it creates an additional layer. If we remove the source directory in a later layer the size of the original COPY layer remains creating a large image, hence we need to squash the layers into one layer. This feature is available with experimental mode enabled. This can be done by modifying /etc/docker/daemon.json to include `"experimental": true,`. You may also be able to pass at the build line as shown below. + Modify the DIR build argument to your local KeyDB repo and update your image tag in the line below ``` -docker build --build-arg DIR=/path/to/keydb-internal -t myImageName:imageTag . +DOCKER_CLI_EXPERIMENTAL=enabled docker build --squash --build-arg DIR=/path/to/keydb-internal -t myImageName:imageTag . ``` Please note that directories are relative to the docker build context. You can use the `-f /path/to/Dockerfile` to specify Dockerfile which will also set the build context, your repo location will be relative to it.