From 1a57af629c4fa4e6bac35602a5373445ca42753b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 22 Mar 2022 17:38:01 +0200 Subject: [PATCH] Split daily CI into smaller chunks (#10469) this should aid find the CI issues with freebsd and macos runs, and also get faster results from valgrind and tls --- .github/workflows/daily.yml | 258 +++++++++++++++++++++++++++++++++--- 1 file changed, 237 insertions(+), 21 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 1039c9553..97aa3b98d 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -200,22 +200,58 @@ jobs: - name: test if: true && !contains(github.event.inputs.skiptests, 'redis') run: | - ./runtest --accurate --verbose --tls --dump-logs ${{github.event.inputs.test_args}} - ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + ./runtest --accurate --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} - name: module api test if: true && !contains(github.event.inputs.skiptests, 'modules') run: | - ./runtest-moduleapi --verbose --tls --dump-logs ${{github.event.inputs.test_args}} - ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + ./runtest-moduleapi --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} - name: sentinel tests if: true && !contains(github.event.inputs.skiptests, 'sentinel') run: | ./runtest-sentinel --tls ${{github.event.inputs.cluster_test_args}} - ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - name: cluster tests if: true && !contains(github.event.inputs.skiptests, 'cluster') run: | ./runtest-cluster --tls ${{github.event.inputs.cluster_test_args}} + + test-ubuntu-tls-no-tls: + runs-on: ubuntu-latest + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'tls') + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + make BUILD_TLS=yes REDIS_CFLAGS='-Werror' + - name: testprep + run: | + sudo apt-get install tcl8.6 tclx tcl-tls + ./utils/gen-test-certs.sh + - name: test + if: true && !contains(github.event.inputs.skiptests, 'redis') + run: | + ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: | + ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: | + ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: | ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-ubuntu-io-threads: @@ -246,7 +282,7 @@ jobs: if: true && !contains(github.event.inputs.skiptests, 'cluster') run: ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes ${{github.event.inputs.cluster_test_args}} - test-valgrind: + test-valgrind-test: runs-on: ubuntu-latest if: | (github.event_name == 'workflow_dispatch' || @@ -271,16 +307,8 @@ jobs: - name: test if: true && !contains(github.event.inputs.skiptests, 'redis') run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} - - name: module api test - if: true && !contains(github.event.inputs.skiptests, 'modules') - run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} - - name: unittest - if: true && !contains(github.event.inputs.skiptests, 'unittest') - run: | - valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all - if grep -q 0x err.txt; then cat err.txt; exit 1; fi - test-valgrind-no-malloc-usable-size: + test-valgrind-misc: runs-on: ubuntu-latest if: | (github.event_name == 'workflow_dispatch' || @@ -297,7 +325,38 @@ jobs: repository: ${{ env.GITHUB_REPOSITORY }} ref: ${{ env.GITHUB_HEAD_REF }} - name: make - run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE" REDIS_CFLAGS='-Werror' + run: make valgrind REDIS_CFLAGS='-Werror -DREDIS_TEST' + - name: testprep + run: | + sudo apt-get update + sudo apt-get install tcl8.6 tclx valgrind -y + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} + - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all + if grep -q 0x err.txt; then cat err.txt; exit 1; fi + + test-valgrind-no-malloc-usable-size-test: + runs-on: ubuntu-latest + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'valgrind') + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DREDIS_TEST" REDIS_CFLAGS='-Werror' - name: testprep run: | sudo apt-get update @@ -305,9 +364,37 @@ jobs: - name: test if: true && !contains(github.event.inputs.skiptests, 'redis') run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} + + test-valgrind-no-malloc-usable-size-misc: + runs-on: ubuntu-latest + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'valgrind') + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DREDIS_TEST" REDIS_CFLAGS='-Werror' + - name: testprep + run: | + sudo apt-get update + sudo apt-get install tcl8.6 tclx valgrind -y - name: module api test if: true && !contains(github.event.inputs.skiptests, 'modules') run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} + - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all + if grep -q 0x err.txt; then cat err.txt; exit 1; fi test-sanitizer-address: runs-on: ubuntu-latest @@ -460,22 +547,61 @@ jobs: - name: test if: true && !contains(github.event.inputs.skiptests, 'redis') run: | - ./runtest --accurate --verbose --tls --dump-logs ${{github.event.inputs.test_args}} - ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + ./runtest --accurate --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} - name: module api test if: true && !contains(github.event.inputs.skiptests, 'modules') run: | - ./runtest-moduleapi --verbose --tls --dump-logs ${{github.event.inputs.test_args}} - ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + ./runtest-moduleapi --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}} - name: sentinel tests if: true && !contains(github.event.inputs.skiptests, 'sentinel') run: | ./runtest-sentinel --tls ${{github.event.inputs.cluster_test_args}} - ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} - name: cluster tests if: true && !contains(github.event.inputs.skiptests, 'cluster') run: | ./runtest-cluster --tls ${{github.event.inputs.cluster_test_args}} + + test-centos7-tls-no-tls: + runs-on: ubuntu-latest + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'tls') + container: centos:7 + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: | + yum -y install centos-release-scl epel-release + yum -y install devtoolset-7 openssl-devel openssl + scl enable devtoolset-7 "make BUILD_TLS=yes REDIS_CFLAGS='-Werror'" + - name: testprep + run: | + yum -y install tcl tcltls tclx + ./utils/gen-test-certs.sh + - name: test + if: true && !contains(github.event.inputs.skiptests, 'redis') + run: | + ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: module api test + if: true && !contains(github.event.inputs.skiptests, 'modules') + run: | + ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}} + - name: sentinel tests + if: true && !contains(github.event.inputs.skiptests, 'sentinel') + run: | + ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + - name: cluster tests + if: true && !contains(github.event.inputs.skiptests, 'cluster') + run: | ./runtest-cluster ${{github.event.inputs.cluster_test_args}} test-macos-latest: @@ -502,9 +628,47 @@ jobs: - name: module api test if: true && !contains(github.event.inputs.skiptests, 'modules') run: ./runtest-moduleapi --verbose --no-latency --dump-logs ${{github.event.inputs.test_args}} + + test-macos-latest-sentinel: + runs-on: macos-latest + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'macos') + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make REDIS_CFLAGS='-Werror' - name: sentinel tests if: true && !contains(github.event.inputs.skiptests, 'sentinel') run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} + + test-macos-latest-cluster: + runs-on: macos-latest + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'macos') + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: make + run: make REDIS_CFLAGS='-Werror' - name: cluster tests if: true && !contains(github.event.inputs.skiptests, 'cluster') run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} @@ -536,7 +700,59 @@ jobs: gmake || exit 1 ; if echo "${{github.event.inputs.skiptests}}" | grep -vq redis ; then ./runtest --verbose --timeout 2400 --no-latency --dump-logs ${{github.event.inputs.test_args}} || exit 1 ; fi ; if echo "${{github.event.inputs.skiptests}}" | grep -vq modules ; then MAKE=gmake ./runtest-moduleapi --verbose --timeout 2400 --no-latency --dump-logs ${{github.event.inputs.test_args}} || exit 1 ; fi ; + + test-freebsd-sentinel: + runs-on: macos-10.15 + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'freebsd') + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: test + uses: vmactions/freebsd-vm@v0.1.6 + with: + usesh: true + sync: rsync + copyback: false + prepare: pkg install -y bash gmake lang/tcl86 lang/tclx + run: > + gmake || exit 1 ; if echo "${{github.event.inputs.skiptests}}" | grep -vq sentinel ; then ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ; + + test-freebsd-cluster: + runs-on: macos-10.15 + if: | + (github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) && !contains(github.event.inputs.skipjobs, 'freebsd') + timeout-minutes: 14400 + steps: + - name: prep + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV + echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + - name: test + uses: vmactions/freebsd-vm@v0.1.6 + with: + usesh: true + sync: rsync + copyback: false + prepare: pkg install -y bash gmake lang/tcl86 lang/tclx + run: > + gmake || exit 1 ; if echo "${{github.event.inputs.skiptests}}" | grep -vq cluster ; then ./runtest-cluster ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ; test-alpine-jemalloc: