From e11c6160471b4203d4055e5fda8ffcdd30216747 Mon Sep 17 00:00:00 2001 From: Sergey Kolosov Date: Thu, 17 Feb 2022 12:22:57 -0800 Subject: [PATCH] build-and-publish.sh script for building and publishing images. --- docker-internal/Dockerfile | 3 +- docker-internal/README.md | 56 ++++++++++++++++- docker-internal/build-and-publish.sh | 94 ++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 4 deletions(-) create mode 100755 docker-internal/build-and-publish.sh diff --git a/docker-internal/Dockerfile b/docker-internal/Dockerfile index 9a849d276..d70099c79 100644 --- a/docker-internal/Dockerfile +++ b/docker-internal/Dockerfile @@ -24,6 +24,7 @@ RUN set -eux; \ gosu nobody true # build KeyDB ARG KEYDB_DIR +ARG MAKE_JOBS="" COPY $KEYDB_DIR /tmp/keydb-internal/ RUN set -eux; \ \ @@ -52,7 +53,7 @@ RUN set -eux; \ grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ - make -j$(nproc) BUILD_TLS=yes NO_LICENSE_CHECK=yes; \ + make -j$([ -z "$MAKE_JOBS" ] && nproc || echo "$MAKE_JOBS") BUILD_TLS=yes NO_LICENSE_CHECK=yes; \ cd src; \ strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ diff --git a/docker-internal/README.md b/docker-internal/README.md index abbb2c0ff..f6a8d7b6d 100644 --- a/docker-internal/README.md +++ b/docker-internal/README.md @@ -21,14 +21,64 @@ DOCKER_CLI_EXPERIMENTAL=enabled docker build --squash --build-arg KEYDB_DIR=. -t Please note that directories are relative to the docker build context. You can use the `-f /path/to/Dockerfile` to specify Dockerfile which will also set the build context, your repo location will be relative to it. +### Pushing + +#### AWS +There is a script ./build-and-publish.sh to build and push image. This script will push images to caching-infra AWS account and caching-infra GCP project. + +If you are pushing to ECR, then you need to add this profile config in your ```~/.aws/config```: + +``` +[profile caching-infra-images-editor] +role_arn = arn:aws:iam::520173307535:role/_Snap_ContainerEditor +output = json +region = us-east-1 +source_profile = default +``` +and to get permission for assuming role [_Snap_ContainerEditor in account caching-infra](https://lease.sc-corp.net/v2/request_access/aws_resources/aws_account?resource=520173307535&roles=%5B_Snap_ContainerEditor%5D). + +Also, if you are using image different from "520173307535.dkr.ecr.us-east-1.amazonaws.com/keydb", then you need to give access to that image to snap-core-prod aws account. That is account where all mesh services are running. Go to your image in AWS Console and add policy: +``` +{ + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "AllowImagePullApp", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::307862320347:root" + }, + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + } + ] +} +``` + +#### GCP + +In order to publish to GCP, you will need to get [Storage Admin Role in project caching-infra](https://lease.sc-corp.net/v2/request_access/gcp_resources/gcp_project?resource=caching-infra&roles=%5Broles/storage.admin%5D) + +For reading image you will need to add your service account to [caching-infra project](https://lease.sc-corp.net/v2/view_iam?resourceType=PRJ&resource=caching-infra) with "Container Registry Service Agent" role. + +#### Example + +``` +DOCKER_CLI_EXPERIMENTAL=enabled ./build-and-publish.sh +``` + ### Troubleshooting If you see error: ``` #11 354.1 g++: fatal error: Killed signal terminated program cc1plus ``` -most likely you are hitting memory constraint. Check -j argument for the "make" command int the output. By default it uses the number of cores on the host. So if that is too high (like 8) and you are building locally -on laptop, try to edit Dockerfile to reduce it to -j2. - +most likely you are hitting memory constraint. If you are running docker build command from the above, then you can try to reduce number of jobs for "make" by adding "--build-arg MAKE_JOBS=" argument to lower value (i.e. 2). If you are running ./build-and-publish.sh you can reduce the number of jobs by passing it in args: +``` +DOCKER_CLI_EXPERIMENTAL=enabled ./build-and-publish.sh -j 2 +``` ## Building the Docker Image Using PAT & Clone diff --git a/docker-internal/build-and-publish.sh b/docker-internal/build-and-publish.sh new file mode 100755 index 000000000..3ece75387 --- /dev/null +++ b/docker-internal/build-and-publish.sh @@ -0,0 +1,94 @@ +#!/bin/sh -e + +set -o pipefail + +# Color for displaying error messages +red=`tput setaf 1` +reset=`tput sgr0` + +# Default value for the command line flag + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +FLAGS_tag="latest" +FLAGS_provider="both" +FLAGS_jobs= + +timestamp() { + date "+%m/%d %H:%M:%S" +} + +print_usage() { + echo "Usage:" + echo " ./{script} [flags]" + echo "" + echo "Flags:" + echo " -p, --provider: name of the cloud provider, can be 'gcp', 'aws' or 'both' (default)" + echo " -t, --tag: tag of the image. Default is 'latest'" + echo " -j, --jobs: the number of jobs when making the build. Default is number of cores on this host" +} + +push_image () { # 1 - image, 2 - repo + export IMAGE_WITH_REPO="$2/$1" + docker tag $1 ${IMAGE_WITH_REPO} + echo "`timestamp` publishing image ${IMAGE_WITH_REPO}" + docker push "${IMAGE_WITH_REPO}" + echo "`timestamp` image ${IMAGE_WITH_REPO} is pushed" +} + +# Processing flags +while [ ! $# -eq 0 ] +do + # The shift below ensures the unprocessed flag is always at $1 + case "$1" in + --help | -h) + print_usage + exit 0 + ;; + --provider | -p) + FLAGS_provider="$2" + if [[ ${FLAGS_provider} != "gcp" ]] && [[ ${FLAGS_provider} != "aws" ]] && [[ ${FLAGS_provider} != "both" ]]; + then + echo "${red} For the flag -p/--provider, only valid values are [\"gcp\", \"aws\", \"both\"]." + exit 1 + fi + shift + ;; + --tag | -t) + FLAGS_tag="$2" + shift + ;; + --jobs | -j) + FLAGS_jobs="$2" + shift + ;; + *) + echo "${red}" + echo "Unrecognized flag: $1." + echo "Run with '--help' flag to see the supported flags." + echo "${reset}" + exit 1 + ;; + esac + shift +done + +echo "`timestamp` building image for ${FLAGS_tag}" +export IMAGE_SUFFIX="keydb:${FLAGS_tag}" +docker build --squash --build-arg KEYDB_DIR=. --build-arg MAKE_JOBS=${FLAGS_jobs} -t keydb:latest -f ${DIR}/Dockerfile ${DIR}/.. + +# Build and publish +if [[ ${FLAGS_provider} == "aws" ]] || [[ ${FLAGS_provider} == "both" ]] +then + export ECR="520173307535.dkr.ecr.us-east-1.amazonaws.com" + echo "`timestamp` Preparing to push image to AWS, ECR: ${ECR}" + aws ecr get-login-password --profile caching-infra-images-editor --region us-east-1 | docker login --username AWS --password-stdin ${ECR} + push_image ${IMAGE_SUFFIX} ${ECR} +fi + +if [[ ${FLAGS_provider} == "gcp" ]] || [[ ${FLAGS_provider} == "both" ]] +then + export GCR="gcr.io/caching-infra" + echo "`timestamp` Preparing to push to GCP, GCR: ${GCR}" + push_image ${IMAGE_SUFFIX} ${GCR} +fi