build-and-publish.sh script for building and publishing images.
This commit is contained in:
Sergey Kolosov
parent
148cc85af0
commit
e11c616047
@ -24,6 +24,7 @@ RUN set -eux; \
|
||||
gosu nobody true
|
||||
# build KeyDB
|
||||
ARG KEYDB_DIR
|
||||
ARG MAKE_JOBS=""
|
||||
COPY $KEYDB_DIR /tmp/keydb-internal/
|
||||
RUN set -eux; \
|
||||
\
|
||||
@ -52,7 +53,7 @@ RUN set -eux; \
|
||||
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \
|
||||
sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \
|
||||
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \
|
||||
make -j$(nproc) BUILD_TLS=yes NO_LICENSE_CHECK=yes; \
|
||||
make -j$([ -z "$MAKE_JOBS" ] && nproc || echo "$MAKE_JOBS") BUILD_TLS=yes NO_LICENSE_CHECK=yes; \
|
||||
cd src; \
|
||||
strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \
|
||||
mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \
|
||||
|
||||
@ -21,14 +21,64 @@ DOCKER_CLI_EXPERIMENTAL=enabled docker build --squash --build-arg KEYDB_DIR=. -t
|
||||
|
||||
Please note that directories are relative to the docker build context. You can use the `-f /path/to/Dockerfile` to specify Dockerfile which will also set the build context, your repo location will be relative to it.
|
||||
|
||||
### Pushing
|
||||
|
||||
#### AWS
|
||||
There is a script ./build-and-publish.sh to build and push image. This script will push images to caching-infra AWS account and caching-infra GCP project.
|
||||
|
||||
If you are pushing to ECR, then you need to add this profile config in your ```~/.aws/config```:
|
||||
|
||||
```
|
||||
[profile caching-infra-images-editor]
|
||||
role_arn = arn:aws:iam::520173307535:role/_Snap_ContainerEditor
|
||||
output = json
|
||||
region = us-east-1
|
||||
source_profile = default
|
||||
```
|
||||
and to get permission for assuming role [_Snap_ContainerEditor in account caching-infra](https://lease.sc-corp.net/v2/request_access/aws_resources/aws_account?resource=520173307535&roles=%5B_Snap_ContainerEditor%5D).
|
||||
|
||||
Also, if you are using image different from "520173307535.dkr.ecr.us-east-1.amazonaws.com/keydb", then you need to give access to that image to snap-core-prod aws account. That is account where all mesh services are running. Go to your image in AWS Console and add policy:
|
||||
```
|
||||
{
|
||||
"Version": "2008-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "AllowImagePullApp",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": "arn:aws:iam::307862320347:root"
|
||||
},
|
||||
"Action": [
|
||||
"ecr:BatchCheckLayerAvailability",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:GetDownloadUrlForLayer"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### GCP
|
||||
|
||||
In order to publish to GCP, you will need to get [Storage Admin Role in project caching-infra](https://lease.sc-corp.net/v2/request_access/gcp_resources/gcp_project?resource=caching-infra&roles=%5Broles/storage.admin%5D)
|
||||
|
||||
For reading image you will need to add your service account to [caching-infra project](https://lease.sc-corp.net/v2/view_iam?resourceType=PRJ&resource=caching-infra) with "Container Registry Service Agent" role.
|
||||
|
||||
#### Example
|
||||
|
||||
```
|
||||
DOCKER_CLI_EXPERIMENTAL=enabled ./build-and-publish.sh
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
If you see error:
|
||||
```
|
||||
#11 354.1 g++: fatal error: Killed signal terminated program cc1plus
|
||||
```
|
||||
most likely you are hitting memory constraint. Check -j argument for the "make" command int the output. By default it uses the number of cores on the host. So if that is too high (like 8) and you are building locally
|
||||
on laptop, try to edit Dockerfile to reduce it to -j2.
|
||||
|
||||
most likely you are hitting memory constraint. If you are running docker build command from the above, then you can try to reduce number of jobs for "make" by adding "--build-arg MAKE_JOBS=<jobs>" argument to lower value (i.e. 2). If you are running ./build-and-publish.sh you can reduce the number of jobs by passing it in args:
|
||||
```
|
||||
DOCKER_CLI_EXPERIMENTAL=enabled ./build-and-publish.sh -j 2
|
||||
```
|
||||
|
||||
## Building the Docker Image Using PAT & Clone
|
||||
|
||||
|
||||
94
docker-internal/build-and-publish.sh
Executable file
94
docker-internal/build-and-publish.sh
Executable file
@ -0,0 +1,94 @@
|
||||
#!/bin/sh -e
|
||||
|
||||
set -o pipefail
|
||||
|
||||
# Color for displaying error messages
|
||||
red=`tput setaf 1`
|
||||
reset=`tput sgr0`
|
||||
|
||||
# Default value for the command line flag
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
|
||||
FLAGS_tag="latest"
|
||||
FLAGS_provider="both"
|
||||
FLAGS_jobs=
|
||||
|
||||
timestamp() {
|
||||
date "+%m/%d %H:%M:%S"
|
||||
}
|
||||
|
||||
print_usage() {
|
||||
echo "Usage:"
|
||||
echo " ./{script} [flags]"
|
||||
echo ""
|
||||
echo "Flags:"
|
||||
echo " -p, --provider: name of the cloud provider, can be 'gcp', 'aws' or 'both' (default)"
|
||||
echo " -t, --tag: tag of the image. Default is 'latest'"
|
||||
echo " -j, --jobs: the number of jobs when making the build. Default is number of cores on this host"
|
||||
}
|
||||
|
||||
push_image () { # 1 - image, 2 - repo
|
||||
export IMAGE_WITH_REPO="$2/$1"
|
||||
docker tag $1 ${IMAGE_WITH_REPO}
|
||||
echo "`timestamp` publishing image ${IMAGE_WITH_REPO}"
|
||||
docker push "${IMAGE_WITH_REPO}"
|
||||
echo "`timestamp` image ${IMAGE_WITH_REPO} is pushed"
|
||||
}
|
||||
|
||||
# Processing flags
|
||||
while [ ! $# -eq 0 ]
|
||||
do
|
||||
# The shift below ensures the unprocessed flag is always at $1
|
||||
case "$1" in
|
||||
--help | -h)
|
||||
print_usage
|
||||
exit 0
|
||||
;;
|
||||
--provider | -p)
|
||||
FLAGS_provider="$2"
|
||||
if [[ ${FLAGS_provider} != "gcp" ]] && [[ ${FLAGS_provider} != "aws" ]] && [[ ${FLAGS_provider} != "both" ]];
|
||||
then
|
||||
echo "${red} For the flag -p/--provider, only valid values are [\"gcp\", \"aws\", \"both\"]."
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--tag | -t)
|
||||
FLAGS_tag="$2"
|
||||
shift
|
||||
;;
|
||||
--jobs | -j)
|
||||
FLAGS_jobs="$2"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "${red}"
|
||||
echo "Unrecognized flag: $1."
|
||||
echo "Run with '--help' flag to see the supported flags."
|
||||
echo "${reset}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "`timestamp` building image for ${FLAGS_tag}"
|
||||
export IMAGE_SUFFIX="keydb:${FLAGS_tag}"
|
||||
docker build --squash --build-arg KEYDB_DIR=. --build-arg MAKE_JOBS=${FLAGS_jobs} -t keydb:latest -f ${DIR}/Dockerfile ${DIR}/..
|
||||
|
||||
# Build and publish
|
||||
if [[ ${FLAGS_provider} == "aws" ]] || [[ ${FLAGS_provider} == "both" ]]
|
||||
then
|
||||
export ECR="520173307535.dkr.ecr.us-east-1.amazonaws.com"
|
||||
echo "`timestamp` Preparing to push image to AWS, ECR: ${ECR}"
|
||||
aws ecr get-login-password --profile caching-infra-images-editor --region us-east-1 | docker login --username AWS --password-stdin ${ECR}
|
||||
push_image ${IMAGE_SUFFIX} ${ECR}
|
||||
fi
|
||||
|
||||
if [[ ${FLAGS_provider} == "gcp" ]] || [[ ${FLAGS_provider} == "both" ]]
|
||||
then
|
||||
export GCR="gcr.io/caching-infra"
|
||||
echo "`timestamp` Preparing to push to GCP, GCR: ${GCR}"
|
||||
push_image ${IMAGE_SUFFIX} ${GCR}
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user