From deef34b2f76a380e5687116acd09ae8d0caf9893 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 12:08:39 -0600 Subject: [PATCH 01/17] setup Machamp ci --- build.yaml | 18 ++++++++++++++++++ build_test.sh | 21 +++++++++++++++++++++ ci.yaml | 20 ++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 build.yaml create mode 100644 build_test.sh create mode 100644 ci.yaml diff --git a/build.yaml b/build.yaml new file mode 100644 index 000000000..868a60c6e --- /dev/null +++ b/build.yaml @@ -0,0 +1,18 @@ +# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284 +version: 1 +machamp: + keydb-build: + # Optional - build counter is linked to the build def + tag_template: 0.0.%build.counter% + # Optional - value in seconds before a build is terminated, default is 3600 seconds + timeout: 3600 + # Optional - update ghe or not, default to true + update_ghe: true + code_coverage: false + # Required + steps: + make-build: + type: cmd + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:6e3e3b99d89522bf25066d6caeb700207429be99fff8c25866c83896bb1fa811 + command: ./build_test.sh diff --git a/build_test.sh b/build_test.sh new file mode 100644 index 000000000..175b5edd9 --- /dev/null +++ b/build_test.sh @@ -0,0 +1,21 @@ +# make the build +make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' + +# gen-cert +./utils/gen-test-certs.sh + +# test-tls +apt-get -y install tcl tcl-tls +./runtest --clients 2 --verbose --tls + +# cluster-test +./runtest-cluster --tls + +# sentinel test +./runtest-sentinel + +# module tests +./runtest-moduleapi + +# rotation test +./runtest-rotation diff --git a/ci.yaml b/ci.yaml new file mode 100644 index 000000000..6c49d6094 --- /dev/null +++ b/ci.yaml @@ -0,0 +1,20 @@ +# Doc: https://wiki.sc-corp.net/display/TOOL/ci.yaml+User+Guide +version: 1 +on: + pull_request: + - workflows: + # All builds that use machamp should use the defined `backend_workflow` + - workflow_type: backend_workflow + # references a build defined in build.yaml + build_name: keydb-build + push: + - branches: [master] + workflows: + - workflow_type: backend_workflow + build_name: keydb-build + arch_types: ["amd64", "arm64"] + cool: + workflow: + - workflow_type: backend_workflow + build_name: keydb-build + arch_types: ["amd64", "arm64"] From fa8815d1971b904ef0ca3b391b51277fed1278af Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 12:24:27 -0600 Subject: [PATCH 02/17] fix typo --- ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.yaml b/ci.yaml index 6c49d6094..861903732 100644 --- a/ci.yaml +++ b/ci.yaml @@ -14,7 +14,7 @@ on: build_name: keydb-build arch_types: ["amd64", "arm64"] cool: - workflow: + workflows: - workflow_type: backend_workflow build_name: keydb-build arch_types: ["amd64", "arm64"] From 4d300194524e90c2e5e91c68f5cfdc576c4070f8 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 12:31:13 -0600 Subject: [PATCH 03/17] make build_test.sh executable --- build_test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build_test.sh diff --git a/build_test.sh b/build_test.sh old mode 100644 new mode 100755 From 50490dece2d9f83da2d4bb3e014763a91d874f85 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 16:46:41 -0600 Subject: [PATCH 04/17] fix the Machamp build --- build.yaml | 2 +- build_test.sh | 2 +- ci.yaml | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/build.yaml b/build.yaml index 868a60c6e..49e163ca3 100644 --- a/build.yaml +++ b/build.yaml @@ -14,5 +14,5 @@ machamp: make-build: type: cmd # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:6e3e3b99d89522bf25066d6caeb700207429be99fff8c25866c83896bb1fa811 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 command: ./build_test.sh diff --git a/build_test.sh b/build_test.sh index 175b5edd9..9c7ccf9a7 100755 --- a/build_test.sh +++ b/build_test.sh @@ -1,11 +1,11 @@ # make the build +git submodule init && git submodule update make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' # gen-cert ./utils/gen-test-certs.sh # test-tls -apt-get -y install tcl tcl-tls ./runtest --clients 2 --verbose --tls # cluster-test diff --git a/ci.yaml b/ci.yaml index 861903732..4cc9c4e16 100644 --- a/ci.yaml +++ b/ci.yaml @@ -13,8 +13,3 @@ on: - workflow_type: backend_workflow build_name: keydb-build arch_types: ["amd64", "arm64"] - cool: - workflows: - - workflow_type: backend_workflow - build_name: keydb-build - arch_types: ["amd64", "arm64"] From ea48d97dfe8dd80b630c5d45df52d281ca0f9452 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 18:03:42 -0600 Subject: [PATCH 05/17] break into tests into steps --- build.yaml | 32 +++++++++++++++++++++++++++++++- build_test.sh | 21 --------------------- machamp_scripts/build.sh | 6 ++++++ 3 files changed, 37 insertions(+), 22 deletions(-) delete mode 100755 build_test.sh create mode 100644 machamp_scripts/build.sh diff --git a/build.yaml b/build.yaml index 49e163ca3..1e14b5ac7 100644 --- a/build.yaml +++ b/build.yaml @@ -15,4 +15,34 @@ machamp: type: cmd # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 - command: ./build_test.sh + command: ./machamp_scripts/build.sh + tls-test: + type: cmd + parent: make-build + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + command: ./runtest --clients 2 --verbose --tls + cluster-test: + type: cmd + parent: make-build + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + command: ./runtest-cluster --tls + sentinel-test: + type: cmd + parent: make-build + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + command: ./runtest-sentinel + module-test: + type: cmd + parent: make-build + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + command: ./runtest-moduleapi + rotation-test: + type: cmd + parent: make-build + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + command: ./runtest-rotation diff --git a/build_test.sh b/build_test.sh deleted file mode 100755 index 9c7ccf9a7..000000000 --- a/build_test.sh +++ /dev/null @@ -1,21 +0,0 @@ -# make the build -git submodule init && git submodule update -make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' - -# gen-cert -./utils/gen-test-certs.sh - -# test-tls -./runtest --clients 2 --verbose --tls - -# cluster-test -./runtest-cluster --tls - -# sentinel test -./runtest-sentinel - -# module tests -./runtest-moduleapi - -# rotation test -./runtest-rotation diff --git a/machamp_scripts/build.sh b/machamp_scripts/build.sh new file mode 100644 index 000000000..0408c0db6 --- /dev/null +++ b/machamp_scripts/build.sh @@ -0,0 +1,6 @@ +# make the build +git submodule init && git submodule update +make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' + +# gen-cert +./utils/gen-test-certs.sh \ No newline at end of file From 0801e2cfb1d850c593187fb5eb813ab892a4f66e Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 18:06:36 -0600 Subject: [PATCH 06/17] make build script executable --- machamp_scripts/build.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 machamp_scripts/build.sh diff --git a/machamp_scripts/build.sh b/machamp_scripts/build.sh old mode 100644 new mode 100755 From 58469d47f3e3240ee79d5df616cdaeed58a39972 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 18:36:40 -0600 Subject: [PATCH 07/17] add multi arch for machamp ci --- ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci.yaml b/ci.yaml index 4cc9c4e16..5614be8e6 100644 --- a/ci.yaml +++ b/ci.yaml @@ -7,6 +7,7 @@ on: - workflow_type: backend_workflow # references a build defined in build.yaml build_name: keydb-build + arch_types: ["amd64", "arm64"] push: - branches: [master] workflows: From fb9335b604e2aef322d88519df8ae5c7fae0e115 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 18:45:19 -0600 Subject: [PATCH 08/17] specify bash on the top of sh file --- machamp_scripts/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/machamp_scripts/build.sh b/machamp_scripts/build.sh index 0408c0db6..27ea80630 100755 --- a/machamp_scripts/build.sh +++ b/machamp_scripts/build.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # make the build git submodule init && git submodule update make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' From bfbea1df82b6ecaf27b0a549c64b3ce2afcb5e56 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 19:06:10 -0600 Subject: [PATCH 09/17] remove multi arch for now it is not working --- ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci.yaml b/ci.yaml index 5614be8e6..26b36c06f 100644 --- a/ci.yaml +++ b/ci.yaml @@ -7,10 +7,8 @@ on: - workflow_type: backend_workflow # references a build defined in build.yaml build_name: keydb-build - arch_types: ["amd64", "arm64"] push: - branches: [master] workflows: - workflow_type: backend_workflow build_name: keydb-build - arch_types: ["amd64", "arm64"] From a64a44172e7621950459ca4cae6e48e5da024864 Mon Sep 17 00:00:00 2001 From: zliang Date: Fri, 22 Apr 2022 19:09:16 -0600 Subject: [PATCH 10/17] -j instead of -j2 to build with the max number of cores instead of just 2 --- machamp_scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machamp_scripts/build.sh b/machamp_scripts/build.sh index 27ea80630..e05a9b3bd 100755 --- a/machamp_scripts/build.sh +++ b/machamp_scripts/build.sh @@ -2,7 +2,7 @@ # make the build git submodule init && git submodule update -make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' +make BUILD_TLS=yes -j$(nproc) KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' # gen-cert ./utils/gen-test-certs.sh \ No newline at end of file From 7876859850781ea38194d0252d92507b09ec7b1e Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 25 Apr 2022 14:49:14 -0600 Subject: [PATCH 11/17] support multi arch build and tests --- build.yaml | 26 +++++++++++++------------- ci.yaml | 2 ++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/build.yaml b/build.yaml index 1e14b5ac7..a291219fa 100644 --- a/build.yaml +++ b/build.yaml @@ -13,36 +13,36 @@ machamp: steps: make-build: type: cmd - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./machamp_scripts/build.sh tls-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 - command: ./runtest --clients 2 --verbose --tls + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + command: ./runtest --clients $(nproc) --verbose --tls cluster-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-cluster --tls sentinel-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-sentinel module-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-moduleapi rotation-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/build/ubuntu/ubuntu-20-04/keydb - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-rotation diff --git a/ci.yaml b/ci.yaml index 26b36c06f..5614be8e6 100644 --- a/ci.yaml +++ b/ci.yaml @@ -7,8 +7,10 @@ on: - workflow_type: backend_workflow # references a build defined in build.yaml build_name: keydb-build + arch_types: ["amd64", "arm64"] push: - branches: [master] workflows: - workflow_type: backend_workflow build_name: keydb-build + arch_types: ["amd64", "arm64"] From 8ae40c84e9c1b1f909cf45614e36a16f33821f65 Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 25 Apr 2022 15:10:00 -0600 Subject: [PATCH 12/17] try remove multi arch for debugging --- ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci.yaml b/ci.yaml index 5614be8e6..26b36c06f 100644 --- a/ci.yaml +++ b/ci.yaml @@ -7,10 +7,8 @@ on: - workflow_type: backend_workflow # references a build defined in build.yaml build_name: keydb-build - arch_types: ["amd64", "arm64"] push: - branches: [master] workflows: - workflow_type: backend_workflow build_name: keydb-build - arch_types: ["amd64", "arm64"] From 27130a084567d13706907d88d79bde07ebf60cdf Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 25 Apr 2022 15:23:51 -0600 Subject: [PATCH 13/17] debug with old img --- build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.yaml b/build.yaml index a291219fa..af068bb8e 100644 --- a/build.yaml +++ b/build.yaml @@ -14,35 +14,35 @@ machamp: make-build: type: cmd # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 command: ./machamp_scripts/build.sh tls-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 command: ./runtest --clients $(nproc) --verbose --tls cluster-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 command: ./runtest-cluster --tls sentinel-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 command: ./runtest-sentinel module-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 command: ./runtest-moduleapi rotation-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 command: ./runtest-rotation From 1d8ca84db4e3918c337aa9fc0de4976bac2eb186 Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 25 Apr 2022 15:32:37 -0600 Subject: [PATCH 14/17] revert back changes --- build.yaml | 12 ++++++------ ci.yaml | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/build.yaml b/build.yaml index af068bb8e..a291219fa 100644 --- a/build.yaml +++ b/build.yaml @@ -14,35 +14,35 @@ machamp: make-build: type: cmd # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./machamp_scripts/build.sh tls-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest --clients $(nproc) --verbose --tls cluster-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-cluster --tls sentinel-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-sentinel module-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-moduleapi rotation-test: type: cmd parent: make-build # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/build/keydb-ubuntu-20-04@sha256:c8e6ecf3c8b385509cc4e0521d7e28d6eef1f70dc7ca0a29f26ec02344870c42 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 command: ./runtest-rotation diff --git a/ci.yaml b/ci.yaml index 26b36c06f..5614be8e6 100644 --- a/ci.yaml +++ b/ci.yaml @@ -7,8 +7,10 @@ on: - workflow_type: backend_workflow # references a build defined in build.yaml build_name: keydb-build + arch_types: ["amd64", "arm64"] push: - branches: [master] workflows: - workflow_type: backend_workflow build_name: keydb-build + arch_types: ["amd64", "arm64"] From ebc293d43a0bfbb4f475ac9b17a5dad0150d9a09 Mon Sep 17 00:00:00 2001 From: zliang Date: Mon, 25 Apr 2022 18:55:01 -0600 Subject: [PATCH 15/17] update builder image --- build.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/build.yaml b/build.yaml index a291219fa..0500f5b7f 100644 --- a/build.yaml +++ b/build.yaml @@ -13,36 +13,36 @@ machamp: steps: make-build: type: cmd - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c command: ./machamp_scripts/build.sh tls-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c command: ./runtest --clients $(nproc) --verbose --tls cluster-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c command: ./runtest-cluster --tls sentinel-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c command: ./runtest-sentinel module-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c command: ./runtest-moduleapi rotation-test: type: cmd parent: make-build - # Dockerfile for the builder img is in https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-latest - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-latest@sha256:abb764c7c19a6b5ffa24db1568b6ab40b16fe1326bff6154f05f774cea990700 + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c command: ./runtest-rotation From b011bcbe93abb4d53367087486c948df0ab71bff Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 27 Apr 2022 17:32:40 +0000 Subject: [PATCH 16/17] Ensure we are responsive during storagecache clears --- src/StorageCache.cpp | 18 ++++++++++++++++-- src/StorageCache.h | 3 ++- src/db.cpp | 4 ++-- src/lazyfree.cpp | 2 +- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/StorageCache.cpp b/src/StorageCache.cpp index 2c6dd42b9..d336e2b3d 100644 --- a/src/StorageCache.cpp +++ b/src/StorageCache.cpp @@ -31,11 +31,25 @@ StorageCache::~StorageCache() dictRelease(m_pdict); } -void StorageCache::clear() +void StorageCache::clear(void(callback)(void*)) { std::unique_lock ul(m_lock); if (m_pdict != nullptr) - dictEmpty(m_pdict, nullptr); + dictEmpty(m_pdict, callback); + m_spstorage->clear(); + m_collisionCount = 0; +} + +void StorageCache::clearAsync() +{ + std::unique_lock ul(m_lock); + if (m_pdict != nullptr) { + dict *dSav = m_pdict; + m_pdict = dictCreate(&dbStorageCacheType, nullptr); + g_pserver->asyncworkqueue->AddWorkFunction([dSav]{ + dictEmpty(dSav, nullptr); + }); + } m_spstorage->clear(); m_collisionCount = 0; } diff --git a/src/StorageCache.h b/src/StorageCache.h index 879b512dc..3c38450fb 100644 --- a/src/StorageCache.h +++ b/src/StorageCache.h @@ -38,7 +38,8 @@ public: return cache; } - void clear(); + void clear(void(callback)(void*)); + void clearAsync(); void insert(sds key, const void *data, size_t cbdata, bool fOverwrite); void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem); void retrieve(sds key, IStorage::callbackSingle fn) const; diff --git a/src/db.cpp b/src/db.cpp index a115c887b..5b67a4198 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -2706,7 +2706,7 @@ void redisDbPersistentData::clear(void(callback)(void*)) delete m_setexpire; m_setexpire = new (MALLOC_LOCAL) expireset(); if (m_spstorage != nullptr) - m_spstorage->clear(); + m_spstorage->clear(callback); dictEmpty(m_pdictTombstone,callback); m_pdbSnapshot = nullptr; } @@ -2926,7 +2926,7 @@ bool redisDbPersistentData::processChanges(bool fSnapshot) if (m_fAllChanged) { if (dictSize(m_pdict) > 0 || m_spstorage->count() > 0) { // in some cases we may have pre-sized the StorageCache's dict, and we don't want clear to ruin it - m_spstorage->clear(); + m_spstorage->clearAsync(); storeDatabase(); } m_fAllChanged = 0; diff --git a/src/lazyfree.cpp b/src/lazyfree.cpp index a05101f2e..fe76b2f4a 100644 --- a/src/lazyfree.cpp +++ b/src/lazyfree.cpp @@ -223,7 +223,7 @@ void redisDbPersistentData::emptyDbAsync() { m_setexpire = new (MALLOC_LOCAL) expireset(); m_pdict = dictCreate(&dbDictType,this); if (m_spstorage != nullptr) - m_spstorage->clear(); + m_spstorage->clearAsync(); if (m_fTrackingChanges) m_fAllChanged = true; atomicIncr(lazyfree_objects,dictSize(oldht1)); From 343c47a935965751bf8be09f3d54128c021b8f0e Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 27 Apr 2022 23:38:55 +0000 Subject: [PATCH 17/17] Ensure recreated tables use the same settings as ones made at boot --- src/storage/rocksdb.cpp | 3 ++- src/storage/rocksdbfactor_internal.h | 2 ++ src/storage/rocksdbfactory.cpp | 19 +++++++++++++------ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index ebe41acc2..7d77cce53 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -132,7 +132,8 @@ size_t RocksDBStorageProvider::clear() auto strName = m_spcolfamily->GetName(); rocksdb::ColumnFamilyHandle *handle = nullptr; - m_spdb->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), strName, &handle); + rocksdb::ColumnFamilyOptions cf_options(m_pfactory->RocksDbOptions()); + m_spdb->CreateColumnFamily(cf_options, strName, &handle); m_spcolfamily = std::shared_ptr(handle); if (!status.ok()) diff --git a/src/storage/rocksdbfactor_internal.h b/src/storage/rocksdbfactor_internal.h index 8a7df2cf5..11862bb7b 100644 --- a/src/storage/rocksdbfactor_internal.h +++ b/src/storage/rocksdbfactor_internal.h @@ -24,6 +24,8 @@ public: virtual size_t filedsRequired() const override; std::string getTempFolder(); + rocksdb::Options RocksDbOptions(); + private: void setVersion(rocksdb::ColumnFamilyHandle*); }; \ No newline at end of file diff --git a/src/storage/rocksdbfactory.cpp b/src/storage/rocksdbfactory.cpp index 8e6ecd0d5..701314d0f 100644 --- a/src/storage/rocksdbfactory.cpp +++ b/src/storage/rocksdbfactory.cpp @@ -39,6 +39,17 @@ IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const return new RocksDBStorageFactory(path, dbnum, rgchConfig, cchConfig); } +rocksdb::Options RocksDBStorageFactory::RocksDbOptions() +{ + rocksdb::Options options = DefaultRocksDBOptions(); + options.max_open_files = filedsRequired(); + options.sst_file_manager = m_pfilemanager; + options.create_if_missing = true; + options.create_missing_column_families = true; + options.info_log_level = rocksdb::ERROR_LEVEL; + return options; +} + RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig) : m_path(dbfile) { @@ -56,14 +67,10 @@ RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, cons m_pfilemanager = std::shared_ptr(rocksdb::NewSstFileManager(rocksdb::Env::Default())); - rocksdb::Options options = DefaultRocksDBOptions(); - options.max_open_files = filedsRequired(); - options.sst_file_manager = m_pfilemanager; - options.create_if_missing = true; - options.create_missing_column_families = true; - options.info_log_level = rocksdb::ERROR_LEVEL; rocksdb::DB *db = nullptr; + auto options = RocksDbOptions(); + for (int idb = 0; idb < dbnum; ++idb) { rocksdb::ColumnFamilyOptions cf_options(options);