From 4fe4b235b2913f07ffbe018c42a19f2f107dd2c0 Mon Sep 17 00:00:00 2001 From: zliang Date: Tue, 8 Aug 2023 19:28:44 -0600 Subject: [PATCH 1/8] add docker build --- build.yaml | 6 ++ machamp_scripts/Dockerfile | 112 +++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 machamp_scripts/Dockerfile diff --git a/build.yaml b/build.yaml index 744ae947a..278793abb 100644 --- a/build.yaml +++ b/build.yaml @@ -46,3 +46,9 @@ machamp: # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c command: ./runtest-rotation + docker: + parent: make-build + type: docker + dockerfile: machamp_scripts/Dockerfile + image_name: keydb # git commit sha will be deafult tag in the final image + workspace_context: ./ diff --git a/machamp_scripts/Dockerfile b/machamp_scripts/Dockerfile new file mode 100644 index 000000000..526517be1 --- /dev/null +++ b/machamp_scripts/Dockerfile @@ -0,0 +1,112 @@ +FROM ubuntu:20.04 +SHELL ["/bin/bash","-c"] +RUN groupadd -r keydb && useradd -r -g keydb keydb +# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true +# build KeyDB +ARG MAKE_JOBS="" +ARG ENABLE_FLASH="" +RUN set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \ + dpkg-dev \ + pkg-config \ + ca-certificates \ + build-essential \ + nasm \ + autotools-dev \ + autoconf \ + libjemalloc-dev \ + tcl \ + tcl-dev \ + uuid-dev \ + libcurl4-openssl-dev \ + libbz2-dev \ + libzstd-dev \ + liblz4-dev \ + libsnappy-dev \ + libssl-dev \ + git; \ + # disable protected mode as it relates to docker + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ + make distclean; \ + make -j$([ -z "$MAKE_JOBS" ] && nproc || echo "$MAKE_JOBS") BUILD_TLS=yes NO_LICENSE_CHECK=yes $([ -z "$ENABLE_FLASH" ] && echo "" || echo "ENABLE_FLASH=$ENABLE_FLASH"); \ + cd src; \ + strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ + mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ + # clean up unused dependencies + echo $savedAptMark; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sed 's:.*/::' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + ; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/*; \ +# create working directories and organize files +RUN \ + mkdir /data && chown keydb:keydb /data; \ + mkdir /flash && chown keydb:keydb /flash; \ + mkdir -p /etc/keydb; \ + cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \ + sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ + sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ + ln -s keydb-cli redis-cli; \ + cd /etc/keydb; \ + ln -s keydb.conf redis.conf; \ + rm -rf /tmp/* +# generate entrypoint script +RUN set -eux; \ + echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ + echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + chmod +x /usr/local/bin/docker-entrypoint.sh +# set remaining image properties +VOLUME /data +WORKDIR /data +ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] +EXPOSE 6379 +CMD ["keydb-server","/etc/keydb/keydb.conf"] From 353695887a5b4aaa82df0144bc8bbbc8e25190fa Mon Sep 17 00:00:00 2001 From: zliang Date: Wed, 9 Aug 2023 11:44:33 -0600 Subject: [PATCH 2/8] fix the working dir in Dockerfile --- machamp_scripts/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/machamp_scripts/Dockerfile b/machamp_scripts/Dockerfile index 526517be1..10904cdd6 100644 --- a/machamp_scripts/Dockerfile +++ b/machamp_scripts/Dockerfile @@ -6,7 +6,7 @@ ENV GOSU_VERSION 1.14 RUN set -eux; \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ + apt-get -o Dpkg::Options::="--force-confnew" install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ rm -rf /var/lib/apt/lists/*; \ dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ @@ -25,11 +25,12 @@ RUN set -eux; \ # build KeyDB ARG MAKE_JOBS="" ARG ENABLE_FLASH="" +COPY . /tmp/keydb-internal RUN set -eux; \ - \ + cd /tmp/keydb-internal; \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ - DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \ + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confnew" install -qqy --no-install-recommends \ dpkg-dev \ pkg-config \ ca-certificates \ From 7bce881287ee92509384c011721f810ced7a664a Mon Sep 17 00:00:00 2001 From: zliang Date: Thu, 10 Aug 2023 11:29:07 -0600 Subject: [PATCH 3/8] add release publish docker image --- build.yaml | 4 ++-- ci.yaml | 10 +++++++++- release.yaml | 31 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 release.yaml diff --git a/build.yaml b/build.yaml index 278793abb..54090e881 100644 --- a/build.yaml +++ b/build.yaml @@ -48,7 +48,7 @@ machamp: command: ./runtest-rotation docker: parent: make-build - type: docker + type: docker # published images can be found in https://console.cloud.google.com/gcr/images/machamp-prod/global/keydb dockerfile: machamp_scripts/Dockerfile image_name: keydb # git commit sha will be deafult tag in the final image - workspace_context: ./ + workspace_context: ./ # # This is the workspace context that your Dockerfile will use to move files around. // If the workspace context is just the root of the repository, you can just use "./". diff --git a/ci.yaml b/ci.yaml index ab346113f..044bb88bb 100644 --- a/ci.yaml +++ b/ci.yaml @@ -2,7 +2,8 @@ version: 1 on: pull_request: - - workflows: + - branches: ['!!main', '*'] # this branch pattern means any branch but not main branch will trigger this pr build + workflows: # All builds that use machamp should use the defined `backend_workflow` - workflow_type: backend_workflow # references a build defined in build.yaml @@ -14,3 +15,10 @@ on: - workflow_type: backend_workflow build_name: keydb-build arch_types: ["amd64", "arm64"] + +# below defines which branch is release branch / release tag +machamp: + releases: + # Note: machamp will only respect the ci.yaml file from default branch for "release branch" definition (most repositories using master/main as default branch) + # https://wiki.sc-corp.net/display/TOOL/Onboard+Machamp+Build+By+ci.yaml+Configuration + - branch_name: ^main$ diff --git a/release.yaml b/release.yaml new file mode 100644 index 000000000..3936322d4 --- /dev/null +++ b/release.yaml @@ -0,0 +1,31 @@ +# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284 +version: 1 +machamp: + keydb-build: # user can define the build name, match with the build name in build.yaml + branches: # defines which branch would trigger this release definition, you can define multiple regex for this field. This also applies to git tags + - ^main$ + post_build_actions: + # https://wiki.sc-corp.net/display/TOOL/Build+Config+Complete+Samples#BuildConfigCompleteSamples-Mybuildhasadockerimage,Iwanttocontrolwhichregionorwhichcloudservicethisimagepublishto,Idonothavepipelinetotrigger + # publish image to gcr repository + publish-image-to-gcr-regions: # user can define the post build action step name + deploy_to: gke + regions: + - gcp-us-central1 + - gcp-us-east1 + - gcp-us-east4 + - gcp-europe-west1 + - gcp-asia-southeast1 + - gcp-asia-south1 + image_name: + - keydb + # publish image to ecr repository + publish-image-to-ecr-regions: # user can define the post build action step name + deploy_to: eks + regions: + - aws-us-west-2 + - aws-us-east-1 + - aws-eu-west-1 + - aws-ap-southeast-1 + - aws-ap-south-1 + image_name: + - keydb From 7a496d3dde6c72cc44fa4b77a1c6d5847c59488c Mon Sep 17 00:00:00 2001 From: zliang Date: Thu, 10 Aug 2023 14:08:37 -0600 Subject: [PATCH 4/8] address intentation and use default release --- build.yaml | 2 +- ci.yaml | 13 ++++++++----- release.yaml | 31 ------------------------------- 3 files changed, 9 insertions(+), 37 deletions(-) delete mode 100644 release.yaml diff --git a/build.yaml b/build.yaml index 54090e881..db1f06b16 100644 --- a/build.yaml +++ b/build.yaml @@ -51,4 +51,4 @@ machamp: type: docker # published images can be found in https://console.cloud.google.com/gcr/images/machamp-prod/global/keydb dockerfile: machamp_scripts/Dockerfile image_name: keydb # git commit sha will be deafult tag in the final image - workspace_context: ./ # # This is the workspace context that your Dockerfile will use to move files around. // If the workspace context is just the root of the repository, you can just use "./". + workspace_context: ./ # This is the workspace context that your Dockerfile will use to move files around. // If the workspace context is just the root of the repository, you can just use "./". diff --git a/ci.yaml b/ci.yaml index 044bb88bb..7e83f89b9 100644 --- a/ci.yaml +++ b/ci.yaml @@ -1,14 +1,17 @@ # Doc: https://wiki.sc-corp.net/display/TOOL/ci.yaml+User+Guide version: 1 on: + # https://wiki.sc-corp.net/display/TOOL/Onboard+Machamp+Build+By+ci.yaml+Configuration + # on pull_request is used for any pr build pull_request: - branches: ['!!main', '*'] # this branch pattern means any branch but not main branch will trigger this pr build workflows: - # All builds that use machamp should use the defined `backend_workflow` - - workflow_type: backend_workflow - # references a build defined in build.yaml - build_name: keydb-build - arch_types: ["amd64", "arm64"] + # All builds that use machamp should use the defined `backend_workflow` + - workflow_type: backend_workflow + # references a build defined in build.yaml + build_name: keydb-build + arch_types: ["amd64", "arm64"] + # on push is used for release branch, meaning: trigger this build when there is commit pushed to this branch push: - branches: [main] workflows: diff --git a/release.yaml b/release.yaml deleted file mode 100644 index 3936322d4..000000000 --- a/release.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284 -version: 1 -machamp: - keydb-build: # user can define the build name, match with the build name in build.yaml - branches: # defines which branch would trigger this release definition, you can define multiple regex for this field. This also applies to git tags - - ^main$ - post_build_actions: - # https://wiki.sc-corp.net/display/TOOL/Build+Config+Complete+Samples#BuildConfigCompleteSamples-Mybuildhasadockerimage,Iwanttocontrolwhichregionorwhichcloudservicethisimagepublishto,Idonothavepipelinetotrigger - # publish image to gcr repository - publish-image-to-gcr-regions: # user can define the post build action step name - deploy_to: gke - regions: - - gcp-us-central1 - - gcp-us-east1 - - gcp-us-east4 - - gcp-europe-west1 - - gcp-asia-southeast1 - - gcp-asia-south1 - image_name: - - keydb - # publish image to ecr repository - publish-image-to-ecr-regions: # user can define the post build action step name - deploy_to: eks - regions: - - aws-us-west-2 - - aws-us-east-1 - - aws-eu-west-1 - - aws-ap-southeast-1 - - aws-ap-south-1 - image_name: - - keydb From b6f91e4c2654c1ea365fe3bb00c619cb377aa9f0 Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 14 Aug 2023 12:03:36 -0600 Subject: [PATCH 5/8] include building keydb statsd module --- machamp_scripts/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/machamp_scripts/Dockerfile b/machamp_scripts/Dockerfile index 10904cdd6..0d3f8716c 100644 --- a/machamp_scripts/Dockerfile +++ b/machamp_scripts/Dockerfile @@ -56,6 +56,7 @@ RUN set -eux; \ make distclean; \ make -j$([ -z "$MAKE_JOBS" ] && nproc || echo "$MAKE_JOBS") BUILD_TLS=yes NO_LICENSE_CHECK=yes $([ -z "$ENABLE_FLASH" ] && echo "" || echo "ENABLE_FLASH=$ENABLE_FLASH"); \ cd src; \ + mv modules/keydb_modstatsd/modstatsd.so /usr/local/lib/; \ strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ # clean up unused dependencies @@ -84,7 +85,8 @@ RUN \ sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ - ln -s keydb-cli redis-cli; \ + echo -e "\nloadmodule /usr/local/lib/modstatsd.so" >> /etc/keydb/keydb.conf; \ + ln -s keydb-cli redis-cli; \ cd /etc/keydb; \ ln -s keydb.conf redis.conf; \ rm -rf /tmp/* From 7d96689ae6aaeb801b1d3c731705f1cea4a086cc Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 14 Aug 2023 16:30:36 -0600 Subject: [PATCH 6/8] fix check in Dockerfile docker-entrypoint.sh --- machamp_scripts/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/machamp_scripts/Dockerfile b/machamp_scripts/Dockerfile index 0d3f8716c..a5f3878ed 100644 --- a/machamp_scripts/Dockerfile +++ b/machamp_scripts/Dockerfile @@ -94,9 +94,8 @@ RUN \ RUN set -eux; \ echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# perpend `keydb-server` if not provided as first argument" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ != "keydb-server" ]' >> /usr/local/bin/docker-entrypoint.sh; \ echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ From e15a2b609c6fe20b9c427c4fa90c767df6eafd03 Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 14 Aug 2023 17:05:37 -0600 Subject: [PATCH 7/8] fix --- machamp_scripts/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machamp_scripts/Dockerfile b/machamp_scripts/Dockerfile index a5f3878ed..9705010fb 100644 --- a/machamp_scripts/Dockerfile +++ b/machamp_scripts/Dockerfile @@ -95,7 +95,7 @@ RUN set -eux; \ echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ echo "# perpend `keydb-server` if not provided as first argument" >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'if [ != "keydb-server" ]' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "${1}" != "keydb-server" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ From 70f2a74ff1bba12a7ba69b0aa60a9ab495c20dcb Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 14 Aug 2023 18:34:05 -0600 Subject: [PATCH 8/8] fix the comment caused stuck docker build --- machamp_scripts/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machamp_scripts/Dockerfile b/machamp_scripts/Dockerfile index 9705010fb..b7f3bf6e4 100644 --- a/machamp_scripts/Dockerfile +++ b/machamp_scripts/Dockerfile @@ -94,7 +94,7 @@ RUN \ RUN set -eux; \ echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# perpend `keydb-server` if not provided as first argument" >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# perpend 'keydb-server' if not provided as first argument" >> /usr/local/bin/docker-entrypoint.sh; \ echo 'if [ "${1}" != "keydb-server" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \